Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
"""
import tensorflow as tf
import tensorflow.contrib.slim as slim
from config import cfg, get_coord_add, get_dataset_size_train, get_num_classes, get_create_inputs
import time
import numpy as np
import sys
import os
import capsnet_em as net
import logging
import daiquiri
daiquiri.setup(level=logging.DEBUG)
logger = daiquiri.getLogger(__name__)
def main(args):
"""Get dataset hyperparameters."""
assert len(args) == 2 and isinstance(args[1], str)
dataset_name = args[1]
logger.info('Using dataset: {}'.format(dataset_name))
coord_add = get_coord_add(dataset_name)
dataset_size = get_dataset_size_train(dataset_name)
num_classes = get_num_classes(dataset_name)
create_inputs = get_create_inputs(dataset_name, is_train=True, epochs=cfg.epoch)
"""Set reproduciable random seed"""
tf.set_random_seed(1234)
with tf.Graph().as_default(), tf.device('/cpu:0'):
from gnocchi import utils
OPTS = [
cfg.IntOpt('aggregation_workers_number',
default=1, min=1,
help='Number of threads to process and store aggregates. '
'Set value roughly equal to number of aggregates to be '
'computed per metric'),
cfg.StrOpt('coordination_url',
secret=True,
help='Coordination driver URL'),
]
LOG = daiquiri.getLogger(__name__)
class CorruptionError(ValueError):
"""Data corrupted, damn it."""
def __init__(self, message):
super(CorruptionError, self).__init__(message)
class CarbonaraBasedStorage(storage.StorageDriver):
def __init__(self, conf, coord=None):
super(CarbonaraBasedStorage, self).__init__(conf)
self.aggregation_workers_number = conf.aggregation_workers_number
if self.aggregation_workers_number == 1:
# NOTE(jd) Avoid using futures at all if we don't want any threads.
import abc
import json
from multiprocessing import Pool
import os
import cv2
import daiquiri
import geopandas as gpd
import numpy as np
from osgeo import gdal
from PIL import Image
from deeposlandia import geometries
logger = daiquiri.getLogger(__name__)
AVAILABLE_DATASETS = ("shapes", "mapillary", "aerial", "tanzania")
GEOGRAPHIC_DATASETS = ("aerial", "tanzania")
class Dataset(metaclass=abc.ABCMeta):
"""Generic class that describes the behavior of a Dataset object: it is
initialized at least with an image size, its label are added always through
the same manner, it can be serialized (save) and deserialized (load)
from/to a `.json` file
Attributes
----------
image_size : int
Size of considered images (height=width), raw images will be resized
#!/usr/bin/env python3
"""Tokenizer for fabric8-analytics tagger."""
import io
import os
import re
import nltk
import daiquiri
import f8a_tagger.defaults as defaults
from f8a_tagger.errors import InstallPrepareError
from f8a_tagger.errors import InvalidInputError
_logger = daiquiri.getLogger(__name__)
class Tokenizer(object):
"""Tokenizer for fabric8-analytics."""
_STOPWORDS_TXT = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'data',
'stopwords.txt')
def __init__(self, stopwords_file=None, ngram_size=1, lemmatizer=None, stemmer=None):
"""Construct.
:param stopwords_file: path to stopwords file or file
:type stopwords_file: str
:param ngram_size: size of ngrams that should be constructed from tokens
:type ngram_size: int
:param lemmatizer: lemmatizer instance to be used
import logging
import daiquiri
import requests
daiquiri.setup(level=logging.INFO)
logger = daiquiri.getLogger(__name__)
class SierraAPI(object):
def __init__(self, api_url, oauth_key, oauth_secret, sess=None):
self.api_url = api_url
self.oauth_key = oauth_key
self.oauth_secret = oauth_secret
self.sess = sess or requests.Session()
self._refresh_auth_token()
# This causes the Session to call ``raise_for_status()`` as soon
# as a response is received, so any failing request will throw an
# exception immediately.
def raise_error(resp, *args, **kwargs):
resp.raise_for_status()
"""Script to start offline evaluation."""
# NOTE: Currently works only with S3DataStore
from uuid import uuid1
import sys
import time
from evaluation_platform.uranus.src.evaluate_data import generate_evaluate_test_s3
import daiquiri
import logging
daiquiri.setup(level=logging.INFO)
_logger = daiquiri.getLogger(__name__)
if __name__ == "__main__":
if len(sys.argv) < 2:
training_data_url = "s3://dev-stack-analysis-clean-data/maven/github/"
result_id = str(uuid1())
_logger.info("No env provided, using default")
_logger.info("Evalutaion result id = {}".format(result_id))
else:
training_data_url = sys.argv[1]
result_id = sys.argv[2]
_logger.info("Env Provided")
_logger.info("S3 URL : {}".format(training_data_url))
t0 = time.time()
_logger.info("Kronos Evaluation started")
generate_evaluate_test_s3(training_url=training_data_url,
""" Utilitary function for Mapillary dataset analysis
"""
import json
import os
import daiquiri
import numpy as np
import pandas as pd
from PIL import Image
from deeposlandia.datasets import GEOGRAPHIC_DATASETS
logger = daiquiri.getLogger(__name__)
def read_config(filename):
"""Read the JSON configuration file.
Parameters
----------
filename : str
Path of the configuration file
Returns
-------
dict
Dataset glossary
"""
with open(filename) as fobj:
import operator
import os
import subprocess
import sys
import tempfile
from urllib import parse
import daiquiri
from git_pull_request import pagure
from git_pull_request import textparse
import github
LOG = daiquiri.getLogger("git-pull-request")
def _run_shell_command(cmd, output=None, raise_on_error=True):
if output is True:
output = subprocess.PIPE
LOG.debug("running %s", cmd)
sub = subprocess.Popen(cmd, stdout=output, stderr=output)
out = sub.communicate()
if raise_on_error and sub.returncode:
raise RuntimeError("%s returned %d" % (cmd, sub.returncode))
if out[0] is not None:
return out[0].strip().decode()
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import daiquiri
from six.moves.urllib.parse import quote
try:
from swiftclient import client as swclient
from swiftclient import utils as swift_utils
except ImportError:
swclient = None
swift_utils = None
from gnocchi import storage
LOG = daiquiri.getLogger(__name__)
def get_connection(conf):
if swclient is None:
raise RuntimeError("python-swiftclient unavailable")
os_options = {
'endpoint_type': conf.swift_endpoint_type,
'service_type': conf.swift_service_type,
'user_domain_name': conf.swift_user_domain_name,
'project_domain_name': conf.swift_project_domain_name,
}
if conf.swift_region:
os_options['region_name'] = conf.swift_region
return swclient.Connection(
"""
import daiquiri
import numpy as np
import pandas as pd
from sklearn.cluster import KMeans
import xgboost as xgb
import seaborn as sns
from matplotlib import pyplot as plt
logger = daiquiri.getLogger("stats")
def preprocess_data_for_clustering(df):
"""Prepare data in order to apply a clustering algorithm
Parameters
----------
df : pandas.DataFrame
Input data, *i.e.* city-related timeseries, supposed to have
`station_id`, `ts` and `nb_bikes` columns
Returns
-------
pandas.DataFrame
Simpified version of `df`, ready to be used for clustering