How to use the maggma.utils.grouper function in maggma

To help you get started, we’ve selected a few maggma examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github materialsintelligence / propnet / propnet / ext / aflow.py View on Github external
# Split up current query matchbook to recover filters
        matchbook_splitter = re.compile(r"(?!\'),(?= 1:
            if len(props) / chunks < 2:
                chunks = len(props) + 1
            query_error = False
            for chunk in grouper(props, (len(props) // chunks) + 1):
                logger.debug('Requesting property chunk {} with {} records'.format(chunks, k))
                props_to_request = list(set(c for c in chunk if c is not None))

                # Exclude orderby keyword if it is not requested in this chunk.
                # If it is included, remove from requested properties to avoid duplication in URI
                orderby_prop = None
                orderby_str = None
                for prop in props_to_request:
                    if orderby_kw.startswith(prop):
                        if orderby_kw.startswith('$'):
                            orderby_str = orderby_kw[1:]
                        else:
                            orderby_str = orderby_kw
                        orderby_prop = prop
                        break
github materialsintelligence / propnet / propnet / dbtools / aflow_ingester.py View on Github external
'all' if not config_['select']
                                                 else config_['select']))

            if config_['select']:
                kws_to_chunk = config_['select']
            else:
                kws_to_chunk = self.keywords

            k = config_['k']
            filter_vals = config_['filter']

            chunk_idx = 0
            chunk_size = 5
            total_chunks = len(kws_to_chunk) // chunk_size + 1

            for chunk in grouper(kws_to_chunk, chunk_size):
                chunk_idx += 1
                logger.debug("Property chunk {} of {}".format(chunk_idx, total_chunks))
                props = [getattr(AFLOW_KWS, c) for c in chunk if c is not None]
                if len(props) == 0:
                    continue
                data_query = self._get_query_obj(config_['catalog'], k,
                                                 config_['exclude'], filter_vals)
                data_query.select(*props)
                success = False
                while not success:
                    try:
                        for entry in data_query:
                            yield entry, config_['targets']
                        success = True
                    except ValueError:
                        if data_query.N == 0:   # Empty query
github materialsintelligence / propnet / propnet / dbtools / separation.py View on Github external
def get_items(self):
        # Borrowed from MapBuilder
        keys = self.propnet_store.distinct('task_id', criteria=self.criteria)
        containers = self.props + ['inputs']
        self.total = len(keys)
        for chunked_keys in grouper(keys, self.chunk_size, None):
            chunked_keys = list(filter(None.__ne__, chunked_keys))
            for doc in list(
                    self.propnet_store.query(
                        criteria={'task_id': {
                            "$in": chunked_keys
                        }},
                        properties=containers + ['task_id'],
                    )):
                yield doc

maggma

Framework to develop datapipelines from files on disk to full dissemenation API

BSD-3-Clause
Latest version published 3 months ago

Package Health Score

78 / 100
Full package analysis