Skip to content

Commit

Permalink
Merge pull request #624 from jjjake/json-patch-test-operator
Browse files Browse the repository at this point in the history 
Add support for JSON Patch test operations
  • Loading branch information
@jjjake
jjjake authored Mar 19, 2024
2 parents c894888 + b10fad5 commit aae9ef2
Show file tree
Hide file tree
Showing 4 changed files with 61 additions and 23 deletions.
19 changes: 13 additions & 6 deletions internetarchive/cli/ia_metadata.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,17 +22,20 @@
ia metadata <identifier>... [--exists | --formats] [--header=<key:value>...]
ia metadata <identifier>... --modify=<key:value>... [--target=<target>]
[--priority=<priority>] [--header=<key:value>...]
[--timeout=<value>]
[--timeout=<value>] [--expect=<key:value>...]
ia metadata <identifier>... --remove=<key:value>... [--priority=<priority>]
[--header=<key:value>...] [--timeout=<value>]
[--expect=<key:value>...]
ia metadata <identifier>... [--append=<key:value>... | --append-list=<key:value>...]
[--priority=<priority>] [--target=<target>]
[--header=<key:value>...] [--timeout=<value>]
[--expect=<key:value>...]
ia metadata <identifier>... --insert=<key:value>... [--priority=<priority>]
[--target=<target>] [--header=<key:value>...]
[--timeout=<value>]
[--timeout=<value>] [--expect=<key:value>...]
ia metadata --spreadsheet=<metadata.csv> [--priority=<priority>]
[--modify=<key:value>...] [--header=<key:value>...] [--timeout=<value>]
[--expect=<key:value>...]
ia metadata --help
options:
Expand All @@ -42,8 +45,10 @@
-t, --target=<target> The metadata target to modify.
-a, --append=<key:value>... Append a string to a metadata element.
-A, --append-list=<key:value>... Append a field to a metadata element.
-i, --insert=<key:value>... Insert a value into a multi-value field given
-i, --insert=<key:value>... Insert a value into a multi-value field given
an index (e.g. `--insert=collection[0]:foo`).
-E, --expect=<key:value>... Test an expectation server-side before applying
patch to item metadata.
-s, --spreadsheet=<metadata.csv> Modify metadata in bulk using a spreadsheet as
input.
-e, --exists Check if an item exists
Expand Down Expand Up @@ -79,13 +84,14 @@

def modify_metadata(item: item.Item, metadata: Mapping, args: Mapping) -> Response:
append = bool(args['--append'])
expect = get_args_dict(args['--expect'])
append_list = bool(args['--append-list'])
insert = bool(args['--insert'])
try:
r = item.modify_metadata(metadata, target=args['--target'], append=append,
priority=args['--priority'], append_list=append_list,
headers=args['--header'], insert=insert,
timeout=args['--timeout'])
expect=expect, priority=args['--priority'],
append_list=append_list, headers=args['--header'],
insert=insert, timeout=args['--timeout'])
assert isinstance(r, Response) # mypy: modify_metadata() -> Request | Response
except ItemLocateError as exc:
print(f'{item.identifier} - error: {exc}', file=sys.stderr)
Expand Down Expand Up @@ -178,6 +184,7 @@ def main(argv: dict, session: session.ArchiveSession) -> None:
str: bool,
'<identifier>': list,
'--modify': list,
'--expect': list,
'--header': Or(None, And(Use(get_args_header_dict), dict),
error='--header must be formatted as --header="key:value"'),
'--append': list,
Expand Down
57 changes: 41 additions & 16 deletions internetarchive/iarequest.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -173,6 +173,7 @@ def __init__(self,
access_key=None,
secret_key=None,
append=None,
expect=None,
append_list=None,
insert=None,
**kwargs):
Expand All @@ -188,6 +189,7 @@ def __init__(self,
self.target = target
self.priority = priority
self.append = append
self.expect = expect
self.append_list = append_list
self.insert = insert

Expand All @@ -210,6 +212,7 @@ def prepare(self):
source_metadata=self.source_metadata,
target=self.target,
append=self.append,
expect=self.expect,
append_list=self.append_list,
insert=self.insert,
)
Expand All @@ -220,14 +223,14 @@ class MetadataPreparedRequest(requests.models.PreparedRequest):
def prepare(self, method=None, url=None, headers=None, files=None, data=None,
params=None, auth=None, cookies=None, hooks=None, metadata={}, # noqa: B006
source_metadata=None, target=None, priority=None, append=None,
append_list=None, insert=None):
expect=None, append_list=None, insert=None):
self.prepare_method(method)
self.prepare_url(url, params)
self.identifier = self.url.split("?")[0].split("/")[-1]
self.prepare_headers(headers)
self.prepare_cookies(cookies)
self.prepare_body(metadata, source_metadata, target, priority, append,
append_list, insert)
append_list, insert, expect)
self.prepare_auth(auth, url)
# Note that prepare_auth must be last to enable authentication schemes
# such as OAuth to work on a fully prepared request.
Expand All @@ -236,7 +239,7 @@ def prepare(self, method=None, url=None, headers=None, files=None, data=None,
self.prepare_hooks(hooks)

def prepare_body(self, metadata, source_metadata, target, priority, append,
append_list, insert):
append_list, insert, expect):
priority = priority or -5

if not source_metadata:
Expand All @@ -261,22 +264,25 @@ def prepare_body(self, metadata, source_metadata, target, priority, append,
patch = prepare_patch(metadata[key],
source_metadata['metadata'],
append,
expect,
append_list,
insert)
except KeyError:
raise ItemLocateError(f"{self.identifier} cannot be located "
"because it is dark or does not exist.")
"because it is dark or does not exist.")
elif key.startswith('files'):
patch = prepare_files_patch(metadata[key],
source_metadata['files'],
append,
key,
append_list,
insert)
insert,
expect)
else:
key = key.split('/')[0]
patch = prepare_target_patch(metadata, source_metadata, append,
target, append_list, key, insert)
target, append_list, key, insert,
expect)
changes.append({'target': key, 'patch': patch})
self.data = {
'-changes': json.dumps(changes),
Expand All @@ -289,17 +295,18 @@ def prepare_body(self, metadata, source_metadata, target, priority, append,
target = 'metadata'
try:
patch = prepare_patch(metadata, source_metadata['metadata'], append,
append_list, insert)
expect, append_list, insert)
except KeyError:
raise ItemLocateError(f"{self.identifier} cannot be located "
"because it is dark or does not exist.")
"because it is dark or does not exist.")
elif 'files' in target:
patch = prepare_files_patch(metadata, source_metadata['files'], append,
target, append_list, insert)
target, append_list, insert, expect)
else:
metadata = {target: metadata}
patch = prepare_target_patch(metadata, source_metadata, append,
target, append_list, target, insert)
target, append_list, target, insert,
expect)
self.data = {
'-patch': json.dumps(patch),
'-target': target,
Expand All @@ -309,7 +316,8 @@ def prepare_body(self, metadata, source_metadata, target, priority, append,
super().prepare_body(self.data, None)


def prepare_patch(metadata, source_metadata, append, append_list=None, insert=None):
def prepare_patch(metadata, source_metadata, append,
expect=None, append_list=None, insert=None):
destination_metadata = source_metadata.copy()
if isinstance(metadata, list):
prepared_metadata = metadata
Expand All @@ -333,11 +341,28 @@ def prepare_patch(metadata, source_metadata, append, append_list=None, insert=No
# Delete metadata items where value is REMOVE_TAG.
destination_metadata = delete_items_from_dict(destination_metadata, 'REMOVE_TAG')
patch = make_patch(source_metadata, destination_metadata).patch
return patch

# Add test operations to patch.
patch_tests = []
for expect_key in expect:
idx = None
if '[' in expect_key:
idx = int(expect_key.split('[')[1].strip(']'))
key = expect_key.split('[')[0]
path = f'/{key}/{idx}'
p_test = {'op': 'test', 'path': path, 'value': expect[expect_key]}
else:
path = f'/{key}'
p_test = {'op': 'test', 'path': path, 'value': expect[expect_key]}

patch_tests.append(p_test)
final_patch = patch_tests + patch

return final_patch


def prepare_target_patch(metadata, source_metadata, append, target, append_list, key,
insert):
insert, expect):

def dictify(lst, key=None, value=None):
if not lst:
Expand All @@ -354,18 +379,18 @@ def dictify(lst, key=None, value=None):
source_metadata = source_metadata.get(_k, {})
else:
source_metadata[_k] = source_metadata.get(_k, {}).get(_k, {})
patch = prepare_patch(metadata, source_metadata, append, append_list, insert)
patch = prepare_patch(metadata, source_metadata, append, expect, append_list, insert)
return patch


def prepare_files_patch(metadata, source_metadata, append, target, append_list,
insert):
insert, expect):
filename = '/'.join(target.split('/')[1:])
for f in source_metadata:
if f.get('name') == filename:
source_metadata = f
break
patch = prepare_patch(metadata, source_metadata, append, append_list, insert)
patch = prepare_patch(metadata, source_metadata, append, expect, append_list, insert)
return patch


Expand Down
6 changes: 6 additions & 0 deletions internetarchive/item.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -770,6 +770,7 @@ def modify_metadata(self,
metadata: Mapping,
target: str | None = None,
append: bool = False,
expect: Mapping | None = None,
append_list: bool = False,
insert: bool = False,
priority: int = 0,
Expand All @@ -794,6 +795,9 @@ def modify_metadata(self,
:param append: Append value to an existing multi-value
metadata field.
:param expect: Provide a dict of expectations to be tested
server-side before applying patch to item metadata.
:param append_list: Append values to an existing multi-value
metadata field. No duplicate values will be added.
Expand All @@ -811,6 +815,7 @@ def modify_metadata(self,
secret_key = secret_key or self.session.secret_key
debug = bool(debug)
headers = headers or {}
expect = expect or {}
request_kwargs = request_kwargs or {}
if timeout:
request_kwargs["timeout"] = float(timeout) # type: ignore
Expand All @@ -835,6 +840,7 @@ def modify_metadata(self,
access_key=access_key,
secret_key=secret_key,
append=append,
expect=expect,
append_list=append_list,
insert=insert)
# Must use Session.prepare_request to make sure session settings
Expand Down
2 changes: 1 addition & 1 deletion tests/test_item.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@
'x-archive-auto-make-bucket': '1',
'authorization': 'LOW a:b',
'accept': '*/*',
'accept-encoding': 'gzip, deflate',
'accept-encoding': 'gzip, deflate, br',
'connection': 'close',
}

Expand Down

0 comments on commit aae9ef2

Please sign in to comment.