How to use the invenio.legacy.bibsched.bibtask.write_message function in invenio

To help you get started, we’ve selected a few invenio examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github inveniosoftware / invenio / invenio / legacy / bibrank / citerank_indexer.py View on Github external
raise Exception
    try:
        filename_ranks = config.get(function, "output_ranks_to_filename")
        max_ranks = config.get(function, "output_rank_limit")
        if not max_ranks.isdigit():
            max_ranks = len_
        else:
            max_ranks = int(max_ranks)
            if max_ranks > len_:
                max_ranks = len_
        ranks = sort_weights(dict_of_ranks)
        write_message("Ranks: %s" % str(ranks), verbose=9)
        write_first_ranks_to_file(ranks, dict_of_ranks, \
                max_ranks, filename_ranks)
    except (ConfigParser.NoOptionError, StandardError):
        write_message("If you want the ranks to be printed in a file you have \
to set output_ranks_to_filename and output_rank_limit \
parameters in the configuration file", verbose=3)
    normalize_weights(dict_of_ranks)
    into_db(dict_of_ranks, rank_method_code)
github inveniosoftware / invenio / invenio / utils / hepdata / __init__.py View on Github external
failed_ids = Queue.Queue()
        recs_queue = Queue.Queue()
        finished_queue = Queue.Queue()
        suspend_resume_queue = Queue.Queue()
        suspend_wait_queue = Queue.Queue()
        main_syn_queue = Queue.Queue()
        task_stats["semaphore"] = threading.Semaphore()

    else:
        insert_queue = SingleThreadQueue()
        correct_queue = SingleThreadQueue()
        failed_ids = SingleThreadQueue()
        recs_queue = SingleThreadQueue()
        task_stats["semaphore"] = None

    write_message("STAGE0: Harvesting data and building the input")


    # feed the input queue
    total_recs = 0
    for recid in hepdata_harvest_get_identifiers():
        recs_queue.put_nowait(recid)
        total_recs += 1
    # spawn necessary number of workers (try not to spawn more than necessary)


    if number_threads > 1:
        for i in xrange(number_threads):
            suspend_wait_queue.put(1)
        ts = [RetrievalWorker(recs_queue, insert_queue, correct_queue, finished_queue, failed_ids, task_stats, suspend_wait_queue, suspend_resume_queue, main_syn_queue, number_threads, force_reupload = force_reupload) for i in xrange(number_threads)]
        update_t = StatusUpdater(total_recs, finished_queue)
        # start all the tasks
github inveniosoftware / invenio / invenio / legacy / bibrank / tag_based_indexer.py View on Github external
def bibrank_engine(run):
    """Run the indexing task.
    Return 1 in case of success and 0 in case of failure.
    """
    startCreate = time.time()

    options["run"] = []
    options["run"].append(run)
    for rank_method_code in options["run"]:
        task_sleep_now_if_required(can_stop_too=True)
        cfg_name = getName(rank_method_code)
        write_message("Running rank method: %s." % cfg_name)
        config = load_config(rank_method_code)
        cfg_short = rank_method_code
        cfg_function = config.get("rank_method", "function") + "_exec"
        cfg_repair_function = config.get("rank_method", "function") + "_repair_exec"
        cfg_name = getName(cfg_short)
        options["validset"] = get_valid_range(rank_method_code)

        if task_get_option("collection"):
            l_of_colls = string.split(task_get_option("collection"), ", ")
            recIDs = perform_request_search(c=l_of_colls)
            recIDs_range = []
            for recID in recIDs:
                recIDs_range.append([recID, recID])
            options["recid_range"] = recIDs_range
        elif task_get_option("id"):
            options["recid_range"] = task_get_option("id")
github EUDAT-B2SHARE / b2share / invenio / legacy / bibsort / engine.py View on Github external
serialized_data_dict = serialize_via_marshal(data_dict)
    serialized_data_dict_ordered = serialize_via_marshal(data_dict_ordered)
    serialized_data_list_sorted = serialize_via_marshal(data_list_sorted)
    write_message('Serialization completed.', verbose=5)
    date = strftime("%Y-%m-%d %H:%M:%S", time.localtime())
    if not update_timestamp:
        try:
            date = run_sql('SELECT last_updated from bsrMETHODDATA WHERE id_bsrMETHOD = %s', (id_method, ))[0][0]
        except IndexError:
            pass # keep the generated date
    write_message("Starting writing the data for method_id=%s " \
                  "to the database (table bsrMETHODDATA)" %id_method, verbose=5)
    try:
        write_message('Deleting old data..', verbose=5)
        run_sql("DELETE FROM bsrMETHODDATA WHERE id_bsrMETHOD = %s", (id_method, ))
        write_message('Inserting new data..', verbose=5)
        run_sql("INSERT into bsrMETHODDATA \
            (id_bsrMETHOD, data_dict, data_dict_ordered, data_list_sorted, last_updated) \
            VALUES (%s, %s, %s, %s, %s)", \
            (id_method, serialized_data_dict, serialized_data_dict_ordered, \
             serialized_data_list_sorted, date, ))
    except Error as err:
        write_message("The error [%s] occured when inserting new bibsort data "\
                      "into bsrMETHODATA table" %err, sys.stderr)
        return False
    write_message('Writing to the bsrMETHODDATA successfully completed.', \
                  verbose=5)
    return True
github inveniosoftware / invenio / invenio / legacy / bibrank / tag_based_indexer.py View on Github external
def check_method(rank_method_code):
    write_message("Checking rank method...")
    if len(fromDB(rank_method_code)) == 0:
        write_message("Rank method not yet executed, please run it to create the necessary data.")
    else:
        if len(add_recIDs_by_date(rank_method_code)) > 0:
            write_message("Records modified, update recommended")
        else:
            write_message("No records modified, update not necessary")
github inveniosoftware / invenio / invenio / legacy / miscutil / dbdump.py View on Github external
dump_database(dump_path, \
                        host=host,
                        port=port,
                        params=params, \
                        compress=compress, \
                        ignore_tables=ignore_tables)
        write_message("Database dump ended")
    finally:
        for queue in active_queues:
            enable_queue(queue)
        if connection and task_get_option('dump_on_slave_helper_mode'):
            write_message("Reattaching slave")
            attach_slave(connection)
    # prune old dump files:
    task_update_progress("Pruning old dump files")
    write_message("Pruning old dump files started")
    _delete_old_dumps(output_dir, output_file_prefix, output_num)
    write_message("Pruning old dump files ended")
    # we are done:
    task_update_progress("Done.")
    return True
github inveniosoftware / invenio / invenio / legacy / bibrank / citerank_indexer.py View on Github external
dates = get_dates_from_file(file_for_dates, dict_of_ids)
    except (ConfigParser.NoOptionError, StandardError) as err:
        write_message("If you want to read the dates from file set up the \
'file_for_dates' variable in the config file [%s]" %err, verbose=3)
    try:
        publication_year_tag = config.get(function, "publication_year_tag")
        dummy = int(publication_year_tag[0:3])
    except (ConfigParser.NoOptionError, StandardError):
        write_message("You need to set up correctly the publication_year_tag \
                      in the cfg file", sys.stderr)
        raise Exception
    try:
        creation_date_tag = config.get(function, "creation_date_tag")
        dummy = int(creation_date_tag[0:3])
    except (ConfigParser.NoOptionError, StandardError):
        write_message("You need to set up correctly the creation_date_tag \
                      in the cfg file", sys.stderr)
        raise Exception
    dates = get_dates_from_db(dict_of_ids, publication_year_tag, \
                              creation_date_tag)
    return dates
github EUDAT-B2SHARE / b2share / invenio / legacy / websession / inveniogc.py View on Github external
def clean_cache():
    """Clean the cache for expired and old files."""
    write_message("""CLEANING OF OLD CACHED RSS REQUEST STARTED""")
    rss_cache_dir = "%s/rss/" % CFG_CACHEDIR
    try:
        filenames = os.listdir(rss_cache_dir)
    except OSError:
        filenames = []
    count = 0
    for filename in filenames:
        filename = os.path.join(rss_cache_dir, filename)
        last_update_time = datetime.datetime.fromtimestamp(os.stat(os.path.abspath(filename)).st_mtime)
        if not (datetime.datetime.now() < last_update_time + datetime.timedelta(minutes=CFG_WEBSEARCH_RSS_TTL)):
            try:
                os.remove(filename)
                count += 1
            except OSError as e:
                write_message("Error: %s" % e)
    write_message("""%s rss cache file pruned out of %s.""" % (count, len(filenames)))
github inveniosoftware / invenio / invenio / modules / scheduler / tasklets / bst_weblinkback_updater.py View on Github external
6 send notification email for all pending linkbacks
    @type mode: int
    """
    mode = int(mode)
    if mode == 1:
        write_message("Starting to delete rejected and pending linkbacks URLs on blacklist")
        delete_linkbacks_on_blacklist()
        write_message("Completed to delete rejected and pending linkbacks URLs on blacklist")
    elif mode == 2:
        write_message("Starting to update the page titles of new linkbacks")
        update_linkbacks(1)
        write_message("Completed to update the page titles of new linkbacks")
    elif mode == 3:
        write_message("Starting to update the page titles of old linkbacks")
        update_linkbacks(2)
        write_message("Completed to update the page titles of old linkbacks")
    elif mode == 4:
        write_message("Starting to update manually set page titles")
        update_linkbacks(3)
        write_message("Completed to update manually set page titles")
    elif mode == 5:
        write_message("Starting to detect and disable broken linkbacks")
        update_linkbacks(4)
        write_message("Completed to detect and disable broken linkbacks")
    elif mode == 6:
        write_message("Starting to send notification email")
        send_pending_linkbacks_notification(CFG_WEBLINKBACK_TYPE['TRACKBACK'])
        write_message("Completed to send notification email")
github inveniosoftware / invenio / invenio / modules / scheduler / tasklets / bst_weblinkback_updater.py View on Github external
2 update page titles of new linkbacks
                 3 update page titles of old linkbacks
                 4 update manually set page titles
                 5 detect and disable broken linkbacks
                 6 send notification email for all pending linkbacks
    @type mode: int
    """
    mode = int(mode)
    if mode == 1:
        write_message("Starting to delete rejected and pending linkbacks URLs on blacklist")
        delete_linkbacks_on_blacklist()
        write_message("Completed to delete rejected and pending linkbacks URLs on blacklist")
    elif mode == 2:
        write_message("Starting to update the page titles of new linkbacks")
        update_linkbacks(1)
        write_message("Completed to update the page titles of new linkbacks")
    elif mode == 3:
        write_message("Starting to update the page titles of old linkbacks")
        update_linkbacks(2)
        write_message("Completed to update the page titles of old linkbacks")
    elif mode == 4:
        write_message("Starting to update manually set page titles")
        update_linkbacks(3)
        write_message("Completed to update manually set page titles")
    elif mode == 5:
        write_message("Starting to detect and disable broken linkbacks")
        update_linkbacks(4)
        write_message("Completed to detect and disable broken linkbacks")
    elif mode == 6:
        write_message("Starting to send notification email")
        send_pending_linkbacks_notification(CFG_WEBLINKBACK_TYPE['TRACKBACK'])
        write_message("Completed to send notification email")