How to use quilt3 - 10 common examples

def upload_test_resources(args: Args):
    # Try running the download pipeline
    try:
        # Get test resources dir
        resources_dir = (
            Path(__file__).parent.parent / "aicsimageio" / "tests" / "resources"
        ).resolve(strict=True)

        # Report with directory will be used for upload
        log.info(f"Using contents of directory: {resources_dir}")

        # Create quilt package
        package = Package()
        package.set_dir("resources", resources_dir)

        # Report package contents
        log.info(f"Package contents: {package}")

        # Construct package name
        package_name = "aicsimageio/test_resources"

        # Check for dry run
        if args.dry_run:
            # Attempt to build the package
            built = package.build(package_name)

            # Get resolved save path
            manifest_save_path = Path("upload_manifest.jsonl").resolve()
            with open(manifest_save_path, "w") as manifest_write:

def download_test_resources(args: Args):
    # Try running the download pipeline
    try:
        # Get test resources dir
        resources_dir = (
            Path(__file__).parent.parent / "aicsimageio" / "tests" / "resources"
        ).resolve()
        resources_dir.mkdir(exist_ok=True)

        # Get quilt package
        package = Package.browse(
            "aicsimageio/test_resources",
            "s3://aics-modeling-packages-test-resources",
            top_hash=args.top_hash
        )

        # Download
        package["resources"].fetch(resources_dir)

        log.info(f"Completed package download.")

    # Catch any exception
    except Exception as e:
        log.error("=============================================")
        if args.debug:
            log.error("\n\n" + traceback.format_exc())
            log.error("=============================================")

if module.__name__ == 'quilt3.data':
            # __path__ must be set even if the package is virtual. Since __path__ will be
            # scanned by all other finders preceding this one in sys.meta_path order, make sure
            # it points to someplace lacking importable objects
            module.__path__ = MODULE_PATH
            return module

        elif len(name_parts) == 3:  # e.g. module.__name__ == quilt3.data.foo
            namespace = name_parts[2]

            # we do not know the name the user will ask for, so populate all valid names
            for pkg in _list_packages():
                pkg_user, pkg_name = pkg.split('/')
                if pkg_user == namespace:
                    module.__dict__[pkg_name] = Package._browse(pkg, registry=registry)

            module.__path__ = MODULE_PATH
            return module

        else:
            assert False

# Wait for a minute for the cluster to fully shutdown
            time.sleep(60)

        #######################################################################

        log.info(f"Completed all tests")
        log.info(f"=" * 80)

        # Ensure save dir exists and save results
        args.save_path.parent.mkdir(parents=True, exist_ok=True)
        with open(args.save_path, "w") as write_out:
            json.dump(all_results, write_out)

        # Construct and push package
        if args.upload:
            p = Package()
            p.set("results.json", args.save_path)
            p.push(
                "aicsimageio/benchmarks",
                "s3://aics-modeling-packages-test-resources",
                message=f"aicsimageio version: {aicsimageio.__version__}",
            )

    # Catch any exception
    except Exception as e:
        log.error("=============================================")
        if args.debug:
            log.error("\n\n" + traceback.format_exc())
            log.error("=============================================")
        log.error("\n\n" + str(e) + "\n")
        log.error("=============================================")
        sys.exit(1)

"""
        name_parts = module.__name__.split('.')
        registry = get_from_config('default_local_registry')

        if module.__name__ == 'quilt3.data':
            # __path__ must be set even if the package is virtual. Since __path__ will be
            # scanned by all other finders preceding this one in sys.meta_path order, make sure
            # it points to someplace lacking importable objects
            module.__path__ = MODULE_PATH
            return module

        elif len(name_parts) == 3:  # e.g. module.__name__ == quilt3.data.foo
            namespace = name_parts[2]

            # we do not know the name the user will ask for, so populate all valid names
            for pkg in _list_packages():
                pkg_user, pkg_name = pkg.split('/')
                if pkg_user == namespace:
                    module.__dict__[pkg_name] = Package._browse(pkg, registry=registry)

            module.__path__ = MODULE_PATH
            return module

        else:
            assert False

def exec_module(cls, module):
        """
        Module executor.
        """
        name_parts = module.__name__.split('.')
        registry = get_from_config('default_local_registry')

        if module.__name__ == 'quilt3.data':
            # __path__ must be set even if the package is virtual. Since __path__ will be
            # scanned by all other finders preceding this one in sys.meta_path order, make sure
            # it points to someplace lacking importable objects
            module.__path__ = MODULE_PATH
            return module

        elif len(name_parts) == 3:  # e.g. module.__name__ == quilt3.data.foo
            namespace = name_parts[2]

            # we do not know the name the user will ask for, so populate all valid names
            for pkg in _list_packages():
                pkg_user, pkg_name = pkg.split('/')
                if pkg_user == namespace:
                    module.__dict__[pkg_name] = Package._browse(pkg, registry=registry)

import quilt3
from appdirs import user_data_dir
from requests.exceptions import Timeout
from shapely import wkb, wkt

appname = "geosnap"
appauthor = "geosnap"
data_dir = user_data_dir(appname, appauthor)
if not os.path.exists(data_dir):
    pathlib.Path(data_dir).mkdir(parents=True, exist_ok=True)

# look for local storage and create if missing
try:
    from quilt3.data.geosnap_data import storage
except ImportError:
    storage = quilt3.Package()


class _Map(dict):
    """tabbable dict."""

    def __init__(self, *args, **kwargs):
        super(_Map, self).__init__(*args, **kwargs)
        for arg in args:
            if isinstance(arg, dict):
                for k, v in arg.iteritems():
                    self[k] = v

        if kwargs:
            for k, v in kwargs.iteritems():
                self[k] = v

You can push a dataset with the same name multiple times to the same bucket multiple times as instead of
        overriding a prior dataset, Quilt simply creates a new dataset version. Please refer to Quilt documentation for
        more details: https://docs.quiltdata.com

        :param push_uri: The S3 bucket uri to push to. Example: "s3://quilt-jacksonb"
        :param message: An optional message to attach to that version of the dataset.
        :param attach_associates: Boolean option to attach associates as metadata to each file. Associates are used
            to retain quick navigation between related files.
        :return: The built and optionally pushed quilt3.Package.
        """
        # Confirm name matches approved pattern
        # We previously checked during init, but the name could have been changed
        name = self.return_or_raise_approved_name(self.name)

        # Create empty package
        pkg = quilt3.Package()

        # Write any extra files to tempdir to send to the build
        with tempfile.TemporaryDirectory() as tmpdir:
            # Set all referenced files
            text = self.readme.text
            for rf in self.readme.referenced_files:
                replaced = f"referenced_files/{rf.resolved.name}"
                text = text.replace(rf.target, replaced)
                pkg.set(replaced, str(rf.resolved))

            # Write the updated readme to temp
            readme_pk = Path(tmpdir, "README.md")
            with open(readme_pk, "w") as readme_write:
                readme_write.write(text)

            # Set the readme

type
        pandas.DataFrame or geopandas.GeoDataFrame
            2010 blocks as a geodataframe or as a dataframe with geometry
            stored as well-known binary on the 'wkb' column.

        """
        try:  # if any of these aren't found, stream them insteead
            from quilt3.data.census import blocks_2010
        except ImportError:
            warn(
                "Unable to locate local census 2010 block data. Streaming instead.\n"
                "If you plan to use census data repeatedly you can store it locally "
                "with the data.store_blocks_2010 function for better performance"
            )
            try:
                blocks_2010 = quilt3.Package.browse(
                    "census/blocks_2010", "s3://quilt-cgs"
                )

            except Timeout:
                warn(
                    "Unable to locate local census data and unable to reach s3 bucket."
                    "You will be unable to use built-in data during this session. "
                    "If you need these data, please try downloading a local copy "
                    "with the data.store_blocks_2010 function, then restart your "
                    "python kernel and try again."
                )

        if isinstance(states, (str, int)):
            states = [states]
        blks = {}
        for state in states:

_fipstable = pd.read_csv(
    os.path.join(os.path.dirname(os.path.abspath(__file__)), "stfipstable.csv"),
    converters={"FIPS Code": str},
)

appname = "geosnap"
appauthor = "geosnap"
data_dir = user_data_dir(appname, appauthor)
if not os.path.exists(data_dir):
    pathlib.Path(data_dir).mkdir(parents=True, exist_ok=True)

# look for local storage and create if missing
try:
    from quilt3.data.geosnap_data import storage
except ImportError:
    storage = quilt3.Package()


def store_census():
    """Save census data to the local quilt package storage.

    Returns
    -------
    None
        Data will be available in the geosnap.data.datasets and will be used
        in place of streaming data for all census queries. The raster package
        is 3.05 GB.

    """
    quilt3.Package.install("census/tracts_cartographic", "s3://quilt-cgs")
    quilt3.Package.install("census/administrative", "s3://quilt-cgs")