Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
def test_unpack_tell():
stream = io.BytesIO()
messages = [2 ** i - 1 for i in range(65)]
messages += [-(2 ** i) for i in range(1, 64)]
messages += [
b"hello",
b"hello" * 1000,
list(range(20)),
{i: bytes(i) * i for i in range(10)},
{i: bytes(i) * i for i in range(32)},
]
offsets = []
for m in messages:
pack(m, stream)
offsets.append(stream.tell())
stream.seek(0)
unpacker = Unpacker(stream, strict_map_key=False)
for m, o in zip(messages, offsets):
m2 = next(unpacker)
assert m == m2
assert o == unpacker.tell()
for typename in typelist:
type_def = schema[typename]
msgpack.pack(len(type_def), output, use_bin_type=True)
layer_cols = []
for k, v in type_def.items():
layer_cols.append(k)
msgpack.pack(k, output, use_bin_type=True)
if isinstance(v, str):
msgpack.pack(False, output)
msgpack.pack(v, output, use_bin_type=True)
elif isinstance(v, dict):
msgpack.pack(True, output)
msgpack.pack(v["type"], output, use_bin_type=True)
msgpack.pack(v["args"], output, use_bin_type=True)
else:
raise NotImplementedError()
types2columns[typename] = layer_cols
out_texts = BytesIO()
# 4. Write Texts
msgpack.pack(texts, out_texts, use_bin_type=True)
msgpack.pack(out_texts.tell(), output)
output.write(out_texts.getbuffer()[0:out_texts.tell()])
# 5. Write Type data
out_types = BytesIO()
for typename in typelist:
def dump_key_data(key_data, file):
import msgpack
dump_data = (key_data[0].id, tuple(ki.dump() for ki in key_data[1]))
msgpack.pack(dump_data, file)
def save_object(object_, file_path):
def ndarrray_to_list(o, _warned=[False]): # Use a mutlable default arg to hold a fn interal temp var.
if isinstance(o, np.ndarray):
if not _warned[0]:
logger.warning("numpy array {} will be serialized as list".format(o))
_warned[0] = True
return o.tolist()
file_path = os.path.expanduser(file_path)
with open(file_path, 'wb') as fh:
msgpack.pack(object_, fh, use_bin_type=True,default=ndarrray_to_list)
def send_msg(self, msg_dict):
""""Messages are delimited by msgapck itself, no need for Storm
multilang end line.
"""
msgpack.pack(msg_dict, self._output_stream)
self._output_stream.flush()
out_props = BytesIO()
# TODO: Implement extension handling!
msgpack.pack(doc.props, out_props)
msgpack.pack(out_props.tell(), output)
output.write(out_props.getbuffer()[0:out_props.tell()])
# 2. Write Inventory of types
msgpack.pack(typelist, output, use_bin_type=True)
types2columns = {}
# 3. Write Schema
for typename in typelist:
type_def = schema[typename]
msgpack.pack(len(type_def), output, use_bin_type=True)
layer_cols = []
for k, v in type_def.items():
layer_cols.append(k)
msgpack.pack(k, output, use_bin_type=True)
if isinstance(v, str):
msgpack.pack(False, output)
msgpack.pack(v, output, use_bin_type=True)
elif isinstance(v, dict):
msgpack.pack(True, output)
msgpack.pack(v["type"], output, use_bin_type=True)
msgpack.pack(v["args"], output, use_bin_type=True)
else:
raise NotImplementedError()
try:
with open("secrets/known_searches.msgp", "rb") as f:
known_searches = msgpack.unpack(f, raw=False)
except ValueError:
known_searches = {}
if not uri in known_searches:
filters = loop.run_until_complete(
search.adjust_api_and_web_filters("https://allegro.pl" + uri, auth)
)
known_searches[uri] = filters
if settings["search"]["cache_file"]:
with open("secrets/known_searches.msgp", "wb") as f:
msgpack.pack(known_searches, f)
result = loop.run_until_complete(
search.search(known_searches[uri]["api"], auth,
limit=settings["search"]["search_results_limit"])
)
offers = []
if not settings["search"]["only_regular_offers"]:
for offer in result["items"]["promoted"]:
offers.append(offer)
for offer in result["items"]["regular"]:
offers.append(offer)
rss_feed = []
for offer in offers:
description = description_builder(uri, known_searches, offer)
def write_to(self, stream):
return msgpack.pack(self.pack(self.sender), stream)
msgpack.pack(typelist, output, use_bin_type=True)
types2columns = {}
# 3. Write Schema
for typename in typelist:
type_def = schema[typename]
msgpack.pack(len(type_def), output, use_bin_type=True)
layer_cols = []
for k, v in type_def.items():
layer_cols.append(k)
msgpack.pack(k, output, use_bin_type=True)
if isinstance(v, str):
msgpack.pack(False, output)
msgpack.pack(v, output, use_bin_type=True)
elif isinstance(v, dict):
msgpack.pack(True, output)
msgpack.pack(v["type"], output, use_bin_type=True)
msgpack.pack(v["args"], output, use_bin_type=True)
else:
raise NotImplementedError()
types2columns[typename] = layer_cols
out_texts = BytesIO()
# 4. Write Texts
msgpack.pack(texts, out_texts, use_bin_type=True)
msgpack.pack(out_texts.tell(), output)
output.write(out_texts.getbuffer()[0:out_texts.tell()])