Skip to content

Commit

Permalink
s3_management: compute checksum in cloud
Browse files Browse the repository at this point in the history
I.e. file never gets downloaded on the client, which is a nice thing
  • Loading branch information
malfet committed Oct 4, 2023
1 parent 2a7b17d commit 3e2d4a1
Showing 1 changed file with 1 addition and 6 deletions.
7 changes: 1 addition & 6 deletions s3_management/manage.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@
import dataclasses
import functools
import time
import hashlib

from os import path, makedirs
from datetime import datetime
Expand Down Expand Up @@ -363,12 +362,8 @@ def compute_sha256(self) -> None:
for obj in self.objects:
if obj.checksum is not None:
continue
print(f"Computing sha256 for {obj.orig_key} of size {obj.size}")
sha256_sum = hashlib.sha256()
print(f"Updating {obj.orig_key} of size {obj.size} with SHA256 checksum")
s3_obj = BUCKET.Object(key=obj.orig_key)
sha256_sum.update(s3_obj.get()["Body"].read())
digest = sha256_sum.hexdigest()
s3_obj.metadata.update({"checksum-sha256": digest})
s3_obj.copy_from(CopySource={"Bucket": BUCKET.name, "Key": obj.orig_key},
Metadata=s3_obj.metadata, MetadataDirective="REPLACE",
ACL="public-read",
Expand Down

0 comments on commit 3e2d4a1

Please sign in to comment.