Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
que = queue.Queue(-1)
queue_handler = logging.handlers.QueueHandler(que)
smtp_handler = logging.handlers.SMTPHandler(**mdata['handlers']['smtp'])
listener = logging.handlers.QueueListener(que, smtp_handler)
dlog.addHandler(queue_handler)
listener.start()
max_tasks = 10000
numb_task = 9
record = "record.dpgen"
iter_rec = [0, -1]
if os.path.isfile (record) :
with open (record) as frec :
for line in frec :
iter_rec = [int(x) for x in line.split()]
dlog.info ("continue from iter %03d task %02d" % (iter_rec[0], iter_rec[1]))
cont = True
ii = -1
while cont:
ii += 1
iter_name=make_iter_name(ii)
sepline(iter_name,'=')
for jj in range (numb_task) :
if ii * max_tasks + jj <= iter_rec[0] * max_tasks + iter_rec[1] :
continue
task_name="task %02d"%jj
sepline("{} {}".format(iter_name, task_name),'-')
if jj == 0 :
log_iter ("make_train", ii, jj)
make_train (ii, jdata, mdata)
elif jj == 1 :
request.set_PayAsYouGoTargetCapacity("0")
request.set_SpotTargetCapacity(str(self.nchunks_limit))
config = self.generate_config()
request.set_LaunchTemplateConfigs(config)
try:
response = self.client.do_action_with_exception(request)
response = json.loads(response)
with open('apg_id.json', 'w') as fp:
json.dump({'apg_id': response["AutoProvisioningGroupId"]}, fp, indent=4)
return response["AutoProvisioningGroupId"]
except ServerException as e:
dlog.info("create apg failed, err msg: %s" % e)
sys.exit()
except ClientException as e:
dlog.info("create apg failed, err msg: %s" % e)
sys.exit()
def gen_run(args) :
if args.PARAM and args.MACHINE:
if args.debug:
dlog.setLevel(logging.DEBUG)
dlog.info ("start running")
run_iter (args.PARAM, args.MACHINE)
dlog.info ("finished")
if args.MACHINE is not None:
make_vasp_relax(jdata, mdata)
run_vasp_relax(jdata, mdata, disp)
else:
make_vasp_relax(jdata, {"fp_resources":{}})
elif stage == 2 :
dlog.info("Current stage is 2, perturb and scale")
make_scale(jdata)
pert_scaled(jdata)
elif stage == 3 :
dlog.info("Current stage is 3, run a short md")
make_vasp_md(jdata)
if args.MACHINE is not None:
run_vasp_md(jdata, mdata, disp)
elif stage == 4 :
dlog.info("Current stage is 4, collect data")
coll_vasp_md(jdata)
else :
raise RuntimeError("unknown stage %d" % stage)
def make_dispatcher(mdata):
try:
hostname = mdata['hostname']
context_type = 'ssh'
except:
context_type = 'local'
try:
batch_type = mdata['batch']
except:
dlog.info('cannot find key "batch" in machine file, try to use deprecated key "machine_type"')
batch_type = mdata['machine_type']
try:
lazy_local = mdata['lazy_local']
except:
lazy_local = False
if lazy_local and context_type == 'local':
dlog.info('Dispatcher switches to the lazy local mode')
context_type = 'lazy-local'
disp = Dispatcher(mdata, context_type=context_type, batch_type=batch_type)
return disp
def gen_simplify(args):
if args.PARAM and args.MACHINE:
if args.debug:
dlog.setLevel(logging.DEBUG)
dlog.info("start simplifying")
run_iter(args.PARAM, args.MACHINE)
dlog.info("finished")
def _rmtree(self, sftp, remotepath, level=0, verbose = False):
for f in sftp.listdir_attr(remotepath):
rpath = os.path.join(remotepath, f.filename)
if stat.S_ISDIR(f.st_mode):
self._rmtree(sftp, rpath, level=(level + 1))
else:
rpath = os.path.join(remotepath, f.filename)
if verbose: dlog.info('removing %s%s' % (' ' * level, rpath))
sftp.remove(rpath)
if verbose: dlog.info('removing %s%s' % (' ' * level, remotepath))
sftp.rmdir(remotepath)
def make_fp(iter_index, jdata, mdata):
iter_name = make_iter_name(iter_index)
work_path = os.path.join(iter_name, fp_name)
create_path(work_path)
picked_data_path = os.path.join(iter_name, model_devi_name, picked_data_name)
if jdata.get("labeled", False):
dlog.info("already labeled, skip make_fp and link data directly")
os.symlink(os.path.abspath(picked_data_path), os.path.abspath(
os.path.join(work_path, "task.%03d" % 0)))
os.symlink(os.path.abspath(picked_data_path), os.path.abspath(
os.path.join(work_path, "data.%03d" % 0)))
return
systems = get_systems(picked_data_path, jdata)
fp_style = jdata['fp_style']
if 'user_fp_params' in jdata.keys() :
fp_params = jdata['user_fp_params']
else:
fp_params = jdata['fp_params']
jj = 0
for system in systems:
for subsys in system:
sys_data = subsys.data
task_name = "task.%03d.%06d" % (0, jj)