INNER CODE UNIT · Python
splitter
fyoorer/ShadowClone · shadowclone.py:49
def splitter(filename, split_factor):
with open(filename, 'r', encoding='utf8', errors='ignore') as f:
data = f.read()
lines = data.split('\n')
arrays = []
for i in range(0, len(lines), split_factor):
lines_group = lines[i:i+split_factor]
arrays.append(np.array(lines_group))
return arrays
def upload_to_bucket(chunk, keep_extension=False):
chunk_hash = perform_hashing(chunk)
if db.exists(chunk_hash):
# same file already exists in bucket
#printerr("[INFO] This file is already uploaded to bucket. Skipping upload.")
return db.get(chunk_hash)