From fe90aa381d35f3b9a191432e2fe28d22a595dbc5 Mon Sep 17 00:00:00 2001 From: Bart Vanbrabant Date: Mon, 10 Dec 2018 14:10:22 +0100 Subject: [PATCH] Remove snapshot restore functionality (#823) * Remove snapshot restore functionality (#fixes 789) --- CHANGELOG.md | 6 + src/inmanta/agent/agent.py | 162 ----------------- src/inmanta/agent/handler.py | 20 --- src/inmanta/const.py | 2 - src/inmanta/data.py | 122 +------------ src/inmanta/methods.py | 129 -------------- src/inmanta/server/server.py | 187 ------------------- tests/test_cache.py | 2 +- tests/test_cli.py | 9 - tests/test_data.py | 83 --------- tests/test_server_agent.py | 337 +---------------------------------- 11 files changed, 14 insertions(+), 1045 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 48b4ff84ca..78c8a29895 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +v 2018.4 (2018-12-xx) +Changes in this release: +- Various bugfixes and performance enhancements +- Dependency updates +- Removal of snapshot and restore functionality from the server (#789) + v 2018.3 (2018-12-07) Changes in this release: - Various bugfixes and performance enhancements diff --git a/src/inmanta/agent/agent.py b/src/inmanta/agent/agent.py index 1614dcf3d8..699149a3b8 100644 --- a/src/inmanta/agent/agent.py +++ b/src/inmanta/agent/agent.py @@ -16,10 +16,8 @@ Contact: code@inmanta.com """ -import base64 from concurrent.futures.thread import ThreadPoolExecutor import datetime -import hashlib import logging import os import random @@ -598,144 +596,6 @@ def do_run_dryrun(self, version, dry_run_id): self._cache.close_version(version) - @gen.coroutine - def do_restore(self, restore_id, snapshot_id, resources): - with (yield self.ratelimiter.acquire()): - - LOGGER.info("Start a restore %s", restore_id) - - yield self.process._ensure_code(self._env_id, resources[0][1]["model"], - [res[1]["resource_type"] for res in resources]) - - version = resources[0][1]["model"] - self._cache.open_version(version) - - for restore, resource in resources: - start = datetime.datetime.now() - provider = None - try: - data = resource["attributes"] - data["id"] = resource["id"] - resource_obj = Resource.deserialize(data) - provider = yield self.get_provider(resource_obj) - if not hasattr(resource_obj, "allow_restore") or not resource_obj.allow_restore: - yield self.get_client().update_restore(tid=self._env_id, - id=restore_id, - resource_id=str(resource_obj.id), - start=start, - stop=datetime.datetime.now(), - success=False, - error=False, - msg="Resource %s does not allow restore" % resource["id"]) - continue - - try: - yield self.thread_pool.submit(provider.restore, resource_obj, restore["content_hash"]) - yield self.get_client().update_restore(tid=self._env_id, id=restore_id, - resource_id=str(resource_obj.id), - success=True, error=False, - start=start, stop=datetime.datetime.now(), msg="") - except NotImplementedError: - yield self.get_client().update_restore(tid=self._env_id, id=restore_id, - resource_id=str(resource_obj.id), - success=False, error=False, - start=start, stop=datetime.datetime.now(), - msg="The handler for resource " - "%s does not support restores" % resource["id"]) - - except Exception: - LOGGER.exception("Unable to find a handler for %s", resource["id"]) - yield self.get_client().update_restore(tid=self._env_id, id=restore_id, - resource_id=resource_obj.id.resource_str(), - success=False, error=False, - start=start, stop=datetime.datetime.now(), - msg="Unable to find a handler to restore a snapshot of resource %s" % - resource["id"]) - finally: - if provider is not None: - provider.close() - self._cache.close_version(version) - - return 200 - - @gen.coroutine - def do_snapshot(self, snapshot_id, resources): - with (yield self.ratelimiter.acquire()): - LOGGER.info("Start snapshot %s", snapshot_id) - - yield self.process._ensure_code(self._env_id, resources[0]["model"], - [res["resource_type"] for res in resources]) - - version = resources[0]["model"] - self._cache.open_version(version) - - for resource in resources: - start = datetime.datetime.now() - provider = None - try: - data = resource["attributes"] - data["id"] = resource["id"] - resource_obj = Resource.deserialize(data) - provider = yield self.get_provider(resource_obj) - - if not hasattr(resource_obj, "allow_snapshot") or not resource_obj.allow_snapshot: - yield self.get_client().update_snapshot(tid=self._env_id, id=snapshot_id, - resource_id=resource_obj.id.resource_str(), snapshot_data="", - start=start, stop=datetime.datetime.now(), size=0, - success=False, error=False, - msg="Resource %s does not allow snapshots" % resource["id"]) - continue - - try: - result = yield self.thread_pool.submit(provider.snapshot, resource_obj) - if result is not None: - sha1sum = hashlib.sha1() - sha1sum.update(result) - content_id = sha1sum.hexdigest() - yield self.get_client().upload_file(id=content_id, content=base64.b64encode(result).decode("ascii")) - - yield self.get_client().update_snapshot(tid=self._env_id, id=snapshot_id, - resource_id=resource_obj.id.resource_str(), - snapshot_data=content_id, - start=start, stop=datetime.datetime.now(), - size=len(result), success=True, error=False, - msg="") - else: - raise Exception("Snapshot returned no data") - - except NotImplementedError: - yield self.get_client().update_snapshot(tid=self._env_id, id=snapshot_id, error=False, - resource_id=resource_obj.id.resource_str(), - snapshot_data="", - start=start, stop=datetime.datetime.now(), - size=0, success=False, - msg="The handler for resource " - "%s does not support snapshots" % resource["id"]) - except Exception: - LOGGER.exception("An exception occurred while creating the snapshot of %s", resource["id"]) - yield self.get_client().update_snapshot(tid=self._env_id, id=snapshot_id, snapshot_data="", - resource_id=resource_obj.id.resource_str(), error=True, - start=start, - stop=datetime.datetime.now(), - size=0, success=False, - msg="The handler for resource " - "%s does not support snapshots" % resource["id"]) - - except Exception: - LOGGER.exception("Unable to find a handler for %s", resource["id"]) - yield self.get_client().update_snapshot(tid=self._env_id, - id=snapshot_id, snapshot_data="", - resource_id=resource_obj.id.resource_str(), error=False, - start=start, stop=datetime.datetime.now(), - size=0, success=False, - msg="Unable to find a handler for %s" % resource["id"]) - finally: - if provider is not None: - provider.close() - - self._cache.close_version(version) - return 200 - @gen.coroutine def get_facts(self, resource): with (yield self.ratelimiter.acquire()): @@ -977,28 +837,6 @@ def check_storage(self): return dir_map - @protocol.handle(methods.AgentRestore.do_restore, env="tid") - @gen.coroutine - def do_restore(self, env, agent, restore_id, snapshot_id, resources): - """ - Restore a snapshot - """ - if agent not in self._instances: - return 200 - - return (yield self._instances[agent].do_restore(restore_id, snapshot_id, resources)) - - @protocol.handle(methods.AgentSnapshot.do_snapshot, env="tid") - @gen.coroutine - def do_snapshot(self, env, agent, snapshot_id, resources): - """ - Create a snapshot of stateful resources managed by this agent - """ - if agent not in self._instances: - return 200 - - return (yield self._instances[agent].do_snapshot(snapshot_id, resources)) - @protocol.handle(methods.AgentParameterMethod.get_parameter, env="tid") @gen.coroutine def get_facts(self, env, agent, resource): diff --git a/src/inmanta/agent/handler.py b/src/inmanta/agent/handler.py index 75e2008a7c..273713ac0b 100644 --- a/src/inmanta/agent/handler.py +++ b/src/inmanta/agent/handler.py @@ -584,26 +584,6 @@ def available(self, resource: resources.Resource) -> bool: """ return True - def snapshot(self, resource: resources.Resource) -> bytes: - """ - Create a new snapshot and upload it to the server - - :param resource: The state of the resource for which a snapshot is created - :return: The data that needs to be uploaded to the server. This data is passed back to the restore method on - snapshot restore. - """ - raise NotImplementedError() - - def restore(self, resource: resources.Resource, snapshot_id: str) -> None: - """ - Restore a resource from a snapshot. - - :param resource: The resource for which a snapshot needs to be restored. - :param snapshot_id: The id of the "file" on the server that contains the snapshot data. This data can be retrieved - with the :func:`~inmanta.agent.handler.ResourceHandler.get_file` method - """ - raise NotImplementedError() - def get_file(self, hash_id) -> bytes: """ Retrieve a file from the fileserver identified with the given id. The convention is to use the sha1sum of the diff --git a/src/inmanta/const.py b/src/inmanta/const.py index 1437b668ff..5a659d3925 100644 --- a/src/inmanta/const.py +++ b/src/inmanta/const.py @@ -93,8 +93,6 @@ class ResourceAction(Enum): pull = 3 deploy = 4 dryrun = 5 - snapshot = 6 - restore = 7 other = 8 diff --git a/src/inmanta/data.py b/src/inmanta/data.py index b74ddab681..f434c2614c 100644 --- a/src/inmanta/data.py +++ b/src/inmanta/data.py @@ -1260,20 +1260,6 @@ def get(cls, environment, resource_version_id): if value is not None: return cls(from_mongo=True, **value) - @classmethod - @gen.coroutine - def get_with_state(cls, environment, version): - """ - Get all resources from the given version that have "state_id" defined - """ - cursor = cls._coll.find({"environment": environment, "model": version, "attributes.state_id": {"$exists": True}}) - - resources = [] - while (yield cursor.fetch_next): - resources.append(cls(from_mongo=True, **cursor.next_object())) - - return resources - @classmethod def new(cls, environment, resource_version_id, **kwargs): vid = Id.parse_id(resource_version_id) @@ -1477,9 +1463,6 @@ def delete_cascade(self): resources = yield Resource.get_list(environment=self.environment, model=self.version) for res in resources: yield res.delete_cascade() - snaps = yield Snapshot.get_list(environment=self.environment, model=self.version) - for snap in snaps: - yield snap.delete_cascade() yield UnknownParameter.delete_all(environment=self.environment, version=self.version) yield Code.delete_all(environment=self.environment, version=self.version) yield DryRun.delete_all(environment=self.environment, model=self.version) @@ -1624,111 +1607,8 @@ def to_dict(self): return dict_result -class ResourceSnapshot(BaseDocument): - """ - Snapshot of a resource - - :param error Indicates if an error made the snapshot fail - """ - environment = Field(field_type=uuid.UUID, required=True) - snapshot = Field(field_type=uuid.UUID, required=True) - resource_id = Field(field_type=str, required=True) - state_id = Field(field_type=str, required=True) - started = Field(field_type=datetime.datetime, default=None) - finished = Field(field_type=datetime.datetime, default=None) - content_hash = Field(field_type=str) - success = Field(field_type=bool) - error = Field(field_type=bool) - msg = Field(field_type=str) - size = Field(field_type=int) - - -class ResourceRestore(BaseDocument): - """ - A restore of a resource from a snapshot - """ - environment = Field(field_type=uuid.UUID, required=True) - restore = Field(field_type=uuid.UUID, required=True) - state_id = Field(field_type=str) - resource_id = Field(field_type=str) - started = Field(field_type=datetime.datetime, default=None) - finished = Field(field_type=datetime.datetime, default=None) - success = Field(field_type=bool) - error = Field(field_type=bool) - msg = Field(field_type=str) - - -class SnapshotRestore(BaseDocument): - """ - Information about a snapshot restore - """ - environment = Field(field_type=uuid.UUID, required=True) - snapshot = Field(field_type=uuid.UUID, required=True) - started = Field(field_type=datetime.datetime, default=None) - finished = Field(field_type=datetime.datetime, default=None) - resources_todo = Field(field_type=int, default=0) - - @gen.coroutine - def delete_cascade(self): - yield ResourceRestore.delete_all(restore=self.id) - yield self.delete() - - @gen.coroutine - def resource_updated(self): - yield SnapshotRestore._coll.update_one({"_id": self.id}, {"$inc": {"resources_todo": int(-1)}}) - self.resources_todo -= 1 - - now = datetime.datetime.now() - result = yield SnapshotRestore._coll.update_one({"_id": self.id, "resources_todo": 0}, {"$set": {"finished": now}}) - if result.matched_count == 1 and (result.modified_count == 1 or result.modified_count is None): - # modified_count is None for mongodb < 2.6 - self.finished = now - - -class Snapshot(BaseDocument): - """ - A snapshot of an environment - - :param id The id of the snapshot - :param environment A reference to the environment - :param started When was this snapshot started - :param finished When was this snapshot finished - :param total_size The total size of this snapshot - """ - environment = Field(field_type=uuid.UUID, required=True) - model = Field(field_type=int, required=True) - name = Field(field_type=str) - started = Field(field_type=datetime.datetime, default=None) - finished = Field(field_type=datetime.datetime, default=None) - total_size = Field(field_type=int, default=0) - resources_todo = Field(field_type=int, default=0) - - @gen.coroutine - def delete_cascade(self): - yield ResourceSnapshot.delete_all(snapshot=self.id) - restores = yield SnapshotRestore.get_list(snapshot=self.id) - for restore in restores: - yield restore.delete_cascade() - - yield self.delete() - - @gen.coroutine - def resource_updated(self, size): - yield Snapshot._coll.update_one({"_id": self.id}, - {"$inc": {"resources_todo": int(-1), "total_size": size}}) - self.total_size += size - self.resources_todo -= 1 - - now = datetime.datetime.now() - result = yield Snapshot._coll.update_one({"_id": self.id, "resources_todo": 0}, {"$set": {"finished": now}}) - if result.matched_count == 1 and (result.modified_count == 1 or result.modified_count is None): - # modified_count is None for mongodb < 2.6 - self.finished = now - - _classes = [Project, Environment, Parameter, UnknownParameter, AgentProcess, AgentInstance, Agent, Report, Compile, Form, - FormRecord, Resource, ResourceAction, ConfigurationModel, Code, DryRun, ResourceSnapshot, ResourceRestore, - SnapshotRestore, Snapshot] + FormRecord, Resource, ResourceAction, ConfigurationModel, Code, DryRun] def use_motor(motor): diff --git a/src/inmanta/methods.py b/src/inmanta/methods.py index d62b1b7a4a..848b8cf092 100644 --- a/src/inmanta/methods.py +++ b/src/inmanta/methods.py @@ -848,135 +848,6 @@ def get_report(self, id: uuid.UUID): """ -class Snapshot(Method): - """ - Methods to create and restore snapshots - """ - __method_name__ = "snapshot" - - @protocol(operation="GET", index=True, arg_options=ENV_OPTS, client_types=["api"]) - def list_snapshots(self, tid: uuid.UUID): - """ - Create a list of all snapshots - """ - - @protocol(operation="GET", id=True, arg_options=ENV_OPTS, client_types=["api"]) - def get_snapshot(self, tid: uuid.UUID, id: uuid.UUID): - """ - Get details about a snapshot and a list of resources that can be snapshot-ed - """ - - @protocol(operation="POST", index=True, arg_options=ENV_OPTS, client_types=["api"]) - def create_snapshot(self, tid: uuid.UUID, name: str=None): - """ - Request a new snapshot - """ - - @protocol(operation="PUT", id=True, agent_server=True, arg_options=ENV_OPTS, client_types=["agent"]) - def update_snapshot(self, tid: uuid.UUID, id: uuid.UUID, resource_id: str, snapshot_data: str, error: bool, success: bool, - start: datetime.datetime, stop: datetime.datetime, size: int, msg: str=None): - """ - Update a snapshot with the details of a specific resource - - :param tid :The environment the snapshot is created in - :param id: The id of the snapshot to update - :param resource_id: The id of the resource of which a snapshot was created - :param snapshot_data: The hash of the snapshot data that was uploaded to the fileserver - :param start: The time when the snapshot was started - :param stop: The time the snapshot ended - :param size: The size of the snapshot in bytes - :param msg: An optional message about the snapshot - """ - - @protocol(operation="DELETE", id=True, arg_options=ENV_OPTS, client_types=["api"]) - def delete_snapshot(self, tid: uuid.UUID, id: uuid.UUID): - """ - Delete a snapshot - """ - - -class AgentSnapshot(Method): - """ - Snapshot operations performed on the agent - """ - __method_name__ = "agent_snapshot" - - @protocol(operation="POST", server_agent=True, timeout=5, arg_options=AGENT_ENV_OPTS, client_types=[]) - def do_snapshot(self, tid: uuid.UUID, agent: str, snapshot_id: uuid.UUID, resources: list): - """ - Create a snapshot of the requested resource - - :param tid The environment the snapshot is created in - :param agent The name of the agent - :param snapshot_id The id of the snapshot to create - :param resource A list of resources to snapshot - """ - - -class RestoreSnapshot(Method): - """ - Restore a snapshot - """ - __method_name__ = "restore" - - @protocol(operation="POST", index=True, arg_options=ENV_OPTS, client_types=["api"]) - def restore_snapshot(self, tid: uuid.UUID, snapshot: uuid.UUID): - """ - Restore a snapshot - - :param tid: The environment the snapshot is created in - :param snapshot: The id of the snapshot to restore - """ - - @protocol(operation="POST", id=True, agent_server=True, arg_options=ENV_OPTS, client_types=["agent"]) - def update_restore(self, tid: uuid.UUID, id: uuid.UUID, resource_id: str, success: bool, error: bool, msg: str, - start: datetime.datetime, stop: datetime.datetime): - """ - Update the status of a restore - - :param tid: The environment to restore the snapshot in - :param id: The id of the restore - :param resource_id: The state id of the resource that was restored. - """ - - @protocol(operation="GET", index=True, arg_options=ENV_OPTS, client_types=["api"]) - def list_restores(self, tid: uuid.UUID): - """ - List finished and ongoing restores - """ - - @protocol(operation="GET", id=True, arg_options=ENV_OPTS, client_types=["api"]) - def get_restore_status(self, tid: uuid.UUID, id: uuid.UUID): - """ - Get the status of a restore - """ - - @protocol(operation="DELETE", id=True, arg_options=ENV_OPTS, client_types=["api"]) - def delete_restore(self, tid: uuid.UUID, id: uuid.UUID): - """ - Cancel a restore - """ - - -class AgentRestore(Method): - """ - Snapshot operations performed on the agent - """ - __method_name__ = "agent_restore" - - @protocol(operation="POST", server_agent=True, timeout=5, arg_options=AGENT_ENV_OPTS, client_types=[]) - def do_restore(self, tid: uuid.UUID, agent: str, restore_id: uuid.UUID, snapshot_id: uuid.UUID, resources: list): - """ - Create a snapshot of the requested resource - - :param tid: The environment the snapshot is created in - :param agent: The name of the agent - :param restore_id: THe id of the restore operation - :param snapshot_id: The id of the snapshot to restore - :param resource: A list of resources to restore - """ - - class NodeMethod(Method): """ Get a list of all agents diff --git a/src/inmanta/server/server.py b/src/inmanta/server/server.py index 6f4ce7add7..7b23cbcb1a 100644 --- a/src/inmanta/server/server.py +++ b/src/inmanta/server/server.py @@ -775,11 +775,6 @@ def put_version(self, env, version, resources, resource_state, unknowns, version # collect all agents agents.add(res_obj.agent) - # for state handling - if "state_id" in res_dict: - if res_dict["state_id"] == "": - res_dict["state_id"] = res_obj.resource_id - attributes = {} for field, value in res_dict.items(): if field != "id": @@ -1617,188 +1612,6 @@ def get_report(self, compile_id): return 200, {"report": report} - @protocol.handle(methods.Snapshot.list_snapshots, env="tid") - @gen.coroutine - def list_snapshots(self, env): - snapshots = yield data.Snapshot.get_list(environment=env.id) - return 200, {"snapshots": snapshots} - - @protocol.handle(methods.Snapshot.get_snapshot, snapshot_id="id", env="tid") - @gen.coroutine - def get_snapshot(self, env, snapshot_id): - snapshot = yield data.Snapshot.get_by_id(snapshot_id) - if snapshot is None: - return 404, {"message": "The given snapshot id does not exist!"} - snap_dict = snapshot.to_dict() - - resources = yield data.ResourceSnapshot.get_list(snapshot=snapshot.id) - snap_dict["resources"] = [r.to_dict() for r in resources] - return 200, {"snapshot": snap_dict} - - @protocol.handle(methods.Snapshot.create_snapshot, env="tid") - @gen.coroutine - def create_snapshot(self, env, name): - # get the latest deployed configuration model - version = yield data.ConfigurationModel.get_latest_version(env.id) - if version is None: - return 500, {"message": "There is no deployed configuration model to create a snapshot."} - - LOGGER.info("Creating a snapshot from version %s in environment %s", version.version, env.id) - - # create the snapshot - snapshot = data.Snapshot(environment=env.id, model=version.version, started=datetime.datetime.now(), name=name) - yield snapshot.insert() - - # find resources with state - resources_to_snapshot = defaultdict(list) - resource_list = [] - resource_states = yield data.Resource.get_with_state(environment=env.id, version=version.version) - - for rs in resource_states: - if rs.status not in const.UNDEPLOYABLE_STATES: - agent = rs.agent - resources_to_snapshot[agent].append(rs.to_dict()) - resource_list.append(rs.resource_id) - r = data.ResourceSnapshot(environment=env.id, snapshot=snapshot.id, resource_id=rs.resource_id, - state_id=rs.attributes["state_id"]) - yield r.insert() - - if len(resource_list) == 0: - yield snapshot.update_fields(finished=datetime.datetime.now(), total_size=0) - else: - yield snapshot.update_fields(resources_todo=len(resource_list)) - - for agent, resources in resources_to_snapshot.items(): - client = self.get_agent_client(env.id, agent) - if client is not None: - future = client.do_snapshot(env.id, agent, snapshot.id, resources) - self.add_future(future) - - value = snapshot.to_dict() - value["resources"] = resource_list - return 200, {"snapshot": value} - - @protocol.handle(methods.Snapshot.update_snapshot, snapshot_id="id", env="tid") - @gen.coroutine - def update_snapshot(self, env, snapshot_id, resource_id, snapshot_data, start, stop, size, success, error, msg): - with (yield agent_lock.acquire()): - snapshot = yield data.Snapshot.get_by_id(snapshot_id) - if snapshot is None: - return 404, {"message": "Snapshot with id %s does not exist!" % snapshot_id} - - res = yield data.ResourceSnapshot.get_list(environment=env.id, snapshot=snapshot.id, resource_id=resource_id) - - if len(res) == 0: - return 404, {"message": "Resource not found"} - res = res[0] - - yield res.update_fields(content_hash=snapshot_data, started=start, finished=stop, size=size, success=success, - error=error, msg=msg) - - yield snapshot.resource_updated(size) - - return 200 - - @protocol.handle(methods.Snapshot.delete_snapshot, snapshot_id="id", env="tid") - @gen.coroutine - def delete_snapshot(self, env, snapshot_id): - snapshot = yield data.Snapshot.get_by_id(snapshot_id) - if snapshot is None: - return 404, {"message": "Snapshot with id %s does not exist!" % snapshot_id} - - yield snapshot.delete_cascade() - - return 200 - - @protocol.handle(methods.RestoreSnapshot.restore_snapshot, env="tid") - @gen.coroutine - def restore_snapshot(self, env, snapshot): - snapshot = yield data.Snapshot.get_by_id(snapshot) - - if snapshot is None: - return 404, {"message": "Snapshot with id %s does not exist!" % snapshot} - - # get all resources in the snapshot - snap_resources = yield data.ResourceSnapshot.get_list(snapshot=snapshot.id) - - # get all resource that support state in the current environment - env_version = yield data.ConfigurationModel.get_latest_version(env.id) - if env_version is None: - return 500, {"message": "There is no deployed configuration model in this environment."} - - env_resources = yield data.Resource.get_with_state(environment=env.id, version=env_version.version) - env_states = {r.attributes["state_id"]: r for r in env_resources} - - # create a restore object - restore = data.SnapshotRestore(snapshot=snapshot.id, environment=env.id, started=datetime.datetime.now()) - - # find matching resources - restore_list = defaultdict(list) - todo = 0 - for r in snap_resources: - if r.state_id in env_states: - env_res = env_states[r.state_id] - LOGGER.debug("Matching state_id %s to %s, scheduling restore" % (r.state_id, env_res.resource_id)) - restore_list[env_res.agent].append((r.to_dict(), env_res.to_dict())) - - rr = data.ResourceRestore(environment=env.id, restore=restore.id, state_id=r.state_id, - resource_id=env_res.resource_version_id, started=datetime.datetime.now(),) - yield rr.insert() - todo += 1 - - restore.resources_todo = todo - yield restore.insert() - - for agent, resources in restore_list.items(): - client = self.get_agent_client(env.id, agent) - if client is not None: - future = client.do_restore(env.id, agent, restore.id, snapshot.id, resources) - self.add_future(future) - - return 200, {"restore": restore} - - @protocol.handle(methods.RestoreSnapshot.list_restores, env="tid") - @gen.coroutine - def list_restores(self, env): - restores = yield data.SnapshotRestore.get_list(environment=env.id) - return 200, {"restores": restores} - - @protocol.handle(methods.RestoreSnapshot.get_restore_status, restore_id="id", env="tid") - @gen.coroutine - def get_restore_status(self, env, restore_id): - restore = yield data.SnapshotRestore.get_by_id(restore_id) - if restore is None: - return 404, {"message": "The given restore id does not exist!"} - - restore_dict = restore.to_dict() - resources = yield data.ResourceRestore.get_list(restore=restore_id) - restore_dict["resources"] = [x.to_dict() for x in resources] - return 200, {"restore": restore_dict} - - @protocol.handle(methods.RestoreSnapshot.update_restore, restore_id="id", env="tid") - @gen.coroutine - def update_restore(self, env, restore_id, resource_id, success, error, msg, start, stop): - restore = yield data.SnapshotRestore.get_by_id(restore_id) - rr = yield data.ResourceRestore.get_list(environment=env.id, restore=restore.id, resource_id=resource_id) - if len(rr) == 0: - return 404, {"message": "Resource restore not found."} - rr = rr[0] - - yield rr.update_fields(error=error, success=success, started=start, finished=stop, msg=msg) - yield restore.resource_updated() - - return 200 - - @protocol.handle(methods.RestoreSnapshot.delete_restore, restore_id="id", env="tid") - @gen.coroutine - def delete_restore(self, env, restore_id): - restore = yield data.SnapshotRestore.get_by_id(restore_id) - if restore is None: - return 404, {"message": "The given restore id does not exist!"} - - yield restore.delete() - return 200 - @protocol.handle(methods.Decommision.decomission_environment, env="id") @gen.coroutine def decomission_environment(self, env, metadata): diff --git a/tests/test_cache.py b/tests/test_cache.py index c91e25f613..013a7e1731 100644 --- a/tests/test_cache.py +++ b/tests/test_cache.py @@ -33,7 +33,7 @@ class MyResource(Resource): """ A file on a filesystem """ - fields = ("key", "value", "purged", "state_id", "allow_snapshot", "allow_restore") + fields = ("key", "value", "purged") class CacheTests(unittest.TestCase): diff --git a/tests/test_cli.py b/tests/test_cli.py index ec23cc6117..2baf72bd69 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -137,9 +137,6 @@ def test_version(server, client, environment, cli): 'id': 'test::Resource[agent1,key=key1],v=' + version, 'send_event': False, 'purged': False, - 'state_id': '', - 'allow_restore': True, - 'allow_snapshot': True, 'requires': ['test::Resource[agent1,key=key2],v=' + version], }, {'key': 'key2', @@ -148,9 +145,6 @@ def test_version(server, client, environment, cli): 'send_event': False, 'requires': [], 'purged': False, - 'state_id': '', - 'allow_restore': True, - 'allow_snapshot': True, }, {'key': 'key3', 'value': None, @@ -158,9 +152,6 @@ def test_version(server, client, environment, cli): 'send_event': False, 'requires': [], 'purged': True, - 'state_id': '', - 'allow_restore': True, - 'allow_snapshot': True, } ] diff --git a/tests/test_data.py b/tests/test_data.py index b7beeceb1f..b31480500b 100644 --- a/tests/test_data.py +++ b/tests/test_data.py @@ -1089,62 +1089,6 @@ def test_resource_get_requires(data_module): assert resources[0].id == res2.id -@pytest.mark.gen_test -def test_resource_get_with_state(data_module): - project = data.Project(name="test") - yield project.insert() - - env = data.Environment(name="dev", project=project.id, repo_url="", repo_branch="") - yield env.insert() - - # model 1 - version = 1 - cm1 = data.ConfigurationModel(environment=env.id, version=version, date=datetime.datetime.now(), total=1, - version_info={}, released=True, deployed=True) - yield cm1.insert() - - res11 = data.Resource.new(environment=env.id, resource_version_id="std::File[agent1,path=/etc/file1],v=%s" % version, - status=const.ResourceState.deployed, last_deploy=datetime.datetime.now(), - attributes={"path": "/etc/file1", "state_id": "test11"}) - yield res11.insert() - res12 = data.Resource.new(environment=env.id, resource_version_id="std::File[agent1,path=/etc/file2],v=%s" % version, - status=const.ResourceState.deployed, last_deploy=datetime.datetime(2018, 7, 14, 12, 30), - attributes={"path": "/etc/file1"}) - yield res12.insert() - res13 = data.Resource.new(environment=env.id, - resource_version_id="std::File[agent1,path=/etc/file3],v=%s" % version, - status=const.ResourceState.deployed, last_deploy=datetime.datetime(2018, 7, 14, 12, 30), - attributes={"path": "/etc/file1", "state_id": "test13"}) - yield res13.insert() - version += 1 - cm2 = data.ConfigurationModel(environment=env.id, version=version, date=datetime.datetime.now(), total=1, - version_info={}, released=True, deployed=True) - yield cm2.insert() - - res21 = data.Resource.new(environment=env.id, resource_version_id="std::File[agent1,path=/etc/file1],v=%s" % version, - status=const.ResourceState.deployed, last_deploy=datetime.datetime.now(), - attributes={"path": "/etc/file1", "state_id": "test"}) - yield res21.insert() - version += 1 - cm3 = data.ConfigurationModel(environment=env.id, version=version, date=datetime.datetime.now(), total=1, - version_info={}, released=True, deployed=True) - yield cm3.insert() - - res31 = data.Resource.new(environment=env.id, resource_version_id="std::File[agent1,path=/etc/file3],v=%s" % version, - status=const.ResourceState.deployed, last_deploy=datetime.datetime.now(), - attributes={"path": "/etc/file1"}) - yield res31.insert() - - resources = yield data.Resource.get_with_state(env.id, 1) - assert len(resources) == 2 - assert sorted([res11.id, res13.id]) == sorted([x.id for x in resources]) - resources = yield data.Resource.get_with_state(env.id, 2) - assert len(resources) == 1 - assert res21.id == resources[0].id - resources = yield data.Resource.get_with_state(env.id, 3) - assert len(resources) == 0 - - @pytest.mark.gen_test def test_resources_delete_cascade(data_module): project = data.Project(name="test") @@ -1279,30 +1223,3 @@ def test_data_document_recursion(data_module): messages=[data.LogLine.log(logging.INFO, "Successfully stored version %(version)d", version=2)]) yield ra.insert() - - -@pytest.mark.gen_test -def test_snapshot(data_module): - project = data.Project(name="test") - yield project.insert() - - env = data.Environment(name="dev", project=project.id, repo_url="", repo_branch="") - yield env.insert() - - snap = data.Snapshot(environment=env.id, model=1, name="a", started=datetime.datetime.now(), resources_todo=1) - yield snap.insert() - - s = yield data.Snapshot.get_by_id(snap.id) - yield s.resource_updated(10) - assert s.resources_todo == 0 - assert s.total_size == 10 - assert s.finished is not None - - s = yield data.Snapshot.get_by_id(snap.id) - assert s.resources_todo == 0 - assert s.total_size == 10 - assert s.finished is not None - - yield s.delete_cascade() - result = yield data.Snapshot.get_list() - assert len(result) == 0 diff --git a/tests/test_server_agent.py b/tests/test_server_agent.py index cf6ca3be4d..31dfea91fb 100644 --- a/tests/test_server_agent.py +++ b/tests/test_server_agent.py @@ -17,7 +17,6 @@ """ from collections import defaultdict, namedtuple import time -import json import uuid from threading import Condition from itertools import groupby @@ -57,7 +56,7 @@ class MyResource(Resource): """ A file on a filesystem """ - fields = ("key", "value", "purged", "state_id", "allow_snapshot", "allow_restore") + fields = ("key", "value", "purged") @resource("test::Fact", agent="agent", id_attribute="key") class FactResource(Resource): @@ -71,35 +70,35 @@ class FailR(Resource): """ A file on a filesystem """ - fields = ("key", "value", "purged", "state_id", "allow_snapshot", "allow_restore") + fields = ("key", "value", "purged") @resource("test::Wait", agent="agent", id_attribute="key") class WaitR(Resource): """ A file on a filesystem """ - fields = ("key", "value", "purged", "state_id", "allow_snapshot", "allow_restore") + fields = ("key", "value", "purged") @resource("test::Noprov", agent="agent", id_attribute="key") class NoProv(Resource): """ A file on a filesystem """ - fields = ("key", "value", "purged", "state_id", "allow_snapshot", "allow_restore") + fields = ("key", "value", "purged") @resource("test::FailFast", agent="agent", id_attribute="key") class FailFastR(Resource): """ A file on a filesystem """ - fields = ("key", "value", "purged", "state_id", "allow_snapshot", "allow_restore") + fields = ("key", "value", "purged") @resource("test::BadEvents", agent="agent", id_attribute="key") class BadeEventR(Resource): """ A file on a filesystem """ - fields = ("key", "value", "purged", "state_id", "allow_snapshot", "allow_restore") + fields = ("key", "value", "purged") @provider("test::Resource", name="test_resource") class Provider(ResourceHandler): @@ -140,18 +139,6 @@ def do_changes(self, ctx, resource, changes): return changes - def snapshot(self, resource): - return json.dumps({"value": self.get(resource.id.get_agent_name(), resource.key), "metadata": "1234"}).encode() - - def restore(self, resource, snapshot_id): - content = self.get_file(snapshot_id) - if content is None: - return - - data = json.loads(content.decode()) - if "value" in data: - self.set(resource.id.get_agent_name(), resource.key, data["value"]) - def facts(self, ctx, resource): return {"length": len(self.get(resource.id.get_agent_name(), resource.key)), "key1": "value1", "key2": "value2"} @@ -415,9 +402,6 @@ def test_dryrun_and_deploy(io_loop, server_multi, client_multi, resource_contain 'id': 'test::Resource[agent1,key=key1],v=%d' % version, 'send_event': False, 'purged': False, - 'state_id': '', - 'allow_restore': True, - 'allow_snapshot': True, 'requires': ['test::Resource[agent1,key=key2],v=%d' % version], }, {'key': 'key2', @@ -426,9 +410,6 @@ def test_dryrun_and_deploy(io_loop, server_multi, client_multi, resource_contain 'send_event': False, 'requires': [], 'purged': False, - 'state_id': '', - 'allow_restore': True, - 'allow_snapshot': True, }, {'key': 'key3', 'value': None, @@ -436,9 +417,6 @@ def test_dryrun_and_deploy(io_loop, server_multi, client_multi, resource_contain 'send_event': False, 'requires': [], 'purged': True, - 'state_id': '', - 'allow_restore': True, - 'allow_snapshot': True, }, {'key': 'key4', 'value': execute.util.Unknown(source=None), @@ -446,9 +424,6 @@ def test_dryrun_and_deploy(io_loop, server_multi, client_multi, resource_contain 'send_event': False, 'requires': [], 'purged': False, - 'state_id': '', - 'allow_restore': True, - 'allow_snapshot': True, }, {'key': 'key5', 'value': "val", @@ -456,9 +431,6 @@ def test_dryrun_and_deploy(io_loop, server_multi, client_multi, resource_contain 'send_event': False, 'requires': ['test::Resource[agent2,key=key4],v=%d' % version], 'purged': False, - 'state_id': '', - 'allow_restore': True, - 'allow_snapshot': True, }, {'key': 'key6', 'value': "val", @@ -466,9 +438,6 @@ def test_dryrun_and_deploy(io_loop, server_multi, client_multi, resource_contain 'send_event': False, 'requires': ['test::Resource[agent2,key=key5],v=%d' % version], 'purged': False, - 'state_id': '', - 'allow_restore': True, - 'allow_snapshot': True, } ] @@ -581,9 +550,6 @@ def test_deploy_with_undefined(io_loop, server_multi, client_multi, resource_con 'id': 'test::Resource[agent2,key=key1],v=%d' % version, 'send_event': False, 'purged': False, - 'state_id': '', - 'allow_restore': True, - 'allow_snapshot': True, 'requires': [], }, {'key': 'key2', @@ -591,9 +557,6 @@ def test_deploy_with_undefined(io_loop, server_multi, client_multi, resource_con 'id': 'test::Resource[agent2,key=key2],v=%d' % version, 'send_event': False, 'purged': False, - 'state_id': '', - 'allow_restore': True, - 'allow_snapshot': True, 'requires': [], }, {'key': 'key4', @@ -603,9 +566,6 @@ def test_deploy_with_undefined(io_loop, server_multi, client_multi, resource_con 'requires': ['test::Resource[agent2,key=key1],v=%d' % version, 'test::Resource[agent2,key=key2],v=%d' % version], 'purged': False, - 'state_id': '', - 'allow_restore': True, - 'allow_snapshot': True, }, {'key': 'key5', 'value': "val", @@ -613,9 +573,6 @@ def test_deploy_with_undefined(io_loop, server_multi, client_multi, resource_con 'send_event': False, 'requires': ['test::Resource[agent2,key=key4],v=%d' % version], 'purged': False, - 'state_id': '', - 'allow_restore': True, - 'allow_snapshot': True, } ] @@ -718,9 +675,6 @@ def test_server_restart(resource_container, io_loop, server, mongo_db, client): 'id': 'test::Resource[agent1,key=key1],v=%d' % version, 'purged': False, 'send_event': False, - 'state_id': '', - 'allow_restore': True, - 'allow_snapshot': True, 'requires': ['test::Resource[agent1,key=key2],v=%d' % version], }, {'key': 'key2', @@ -729,9 +683,6 @@ def test_server_restart(resource_container, io_loop, server, mongo_db, client): 'requires': [], 'purged': False, 'send_event': False, - 'state_id': '', - 'allow_restore': True, - 'allow_snapshot': True, }, {'key': 'key3', 'value': None, @@ -739,9 +690,6 @@ def test_server_restart(resource_container, io_loop, server, mongo_db, client): 'requires': [], 'purged': True, 'send_event': False, - 'state_id': '', - 'allow_restore': True, - 'allow_snapshot': True, } ] @@ -838,9 +786,6 @@ def test_spontaneous_deploy(resource_container, io_loop, server, client): 'id': 'test::Resource[agent1,key=key1],v=%d' % version, 'purged': False, 'send_event': False, - 'state_id': '', - 'allow_restore': True, - 'allow_snapshot': True, 'requires': ['test::Resource[agent1,key=key2],v=%d' % version], }, {'key': 'key2', @@ -849,9 +794,6 @@ def test_spontaneous_deploy(resource_container, io_loop, server, client): 'requires': [], 'purged': False, 'send_event': False, - 'state_id': '', - 'allow_restore': True, - 'allow_snapshot': True, }, {'key': 'key3', 'value': None, @@ -859,9 +801,6 @@ def test_spontaneous_deploy(resource_container, io_loop, server, client): 'requires': [], 'purged': True, 'send_event': False, - 'state_id': '', - 'allow_restore': True, - 'allow_snapshot': True, } ] @@ -919,9 +858,6 @@ def test_failing_deploy_no_handler(resource_container, io_loop, server, client): 'id': 'test::Noprov[agent1,key=key1],v=%d' % version, 'purged': False, 'send_event': False, - 'state_id': '', - 'allow_restore': True, - 'allow_snapshot': True, 'requires': [], } ] @@ -975,9 +911,6 @@ def test_dual_agent(resource_container, io_loop, server, client, environment): 'id': 'test::Wait[agent1,key=key1],v=%d' % version, 'purged': False, 'send_event': False, - 'state_id': '', - 'allow_restore': True, - 'allow_snapshot': True, 'requires': [] }, {'key': 'key2', @@ -985,9 +918,6 @@ def test_dual_agent(resource_container, io_loop, server, client, environment): 'id': 'test::Wait[agent1,key=key2],v=%d' % version, 'purged': False, 'send_event': False, - 'state_id': '', - 'allow_restore': True, - 'allow_snapshot': True, 'requires': ['test::Wait[agent1,key=key1],v=%d' % version] }, {'key': 'key1', @@ -995,9 +925,6 @@ def test_dual_agent(resource_container, io_loop, server, client, environment): 'id': 'test::Wait[agent2,key=key1],v=%d' % version, 'purged': False, 'send_event': False, - 'state_id': '', - 'allow_restore': True, - 'allow_snapshot': True, 'requires': [] }, {'key': 'key2', @@ -1005,9 +932,6 @@ def test_dual_agent(resource_container, io_loop, server, client, environment): 'id': 'test::Wait[agent2,key=key2],v=%d' % version, 'purged': False, 'send_event': False, - 'state_id': '', - 'allow_restore': True, - 'allow_snapshot': True, 'requires': ['test::Wait[agent2,key=key1],v=%d' % version] }] @@ -1046,114 +970,6 @@ def test_dual_agent(resource_container, io_loop, server, client, environment): myagent.stop() -@pytest.mark.gen_test(timeout=60) -def test_snapshot_restore(resource_container, client, server, io_loop): - """ - create a snapshot and restore it again - """ - resource_container.Provider.reset() - result = yield client.create_project("env-test") - project_id = result.result["project"]["id"] - - result = yield client.create_environment(project_id=project_id, name="dev") - env_id = result.result["environment"]["id"] - - agent = Agent(io_loop, hostname="node1", environment=env_id, agent_map={"agent1": "localhost"}, - code_loader=False) - agent.add_end_point_name("agent1") - agent.start() - yield retry_limited(lambda: len(server.get_endpoint("session")._sessions) == 1, 10) - - resource_container.Provider.set("agent1", "key", "value") - - version = int(time.time()) - - resources = [{'key': 'key', - 'value': 'value', - 'id': 'test::Resource[agent1,key=key],v=%d' % version, - 'requires': [], - 'purged': False, - 'send_event': False, - 'state_id': '', - 'allow_restore': True, - 'allow_snapshot': True, - }, - {'key': 'key2', - 'value': 'value', - 'id': 'test::Resource[agent1,key=key2],v=%d' % version, - 'requires': [], - 'purged': False, - 'send_event': False, - 'state_id': '', - 'allow_restore': True, - 'allow_snapshot': True, - }] - - result = yield client.put_version(tid=env_id, version=version, resources=resources, unknowns=[], version_info={}) - assert result.code == 200 - - # deploy and wait until done - result = yield client.release_version(env_id, version, True) - assert result.code == 200 - - result = yield client.get_version(env_id, version) - assert result.code == 200 - while (result.result["model"]["total"] - result.result["model"]["done"]) > 0: - result = yield client.get_version(env_id, version) - yield gen.sleep(0.1) - - assert result.result["model"]["done"] == len(resources) - - # create a snapshot - result = yield client.create_snapshot(env_id, "snap1") - assert result.code == 200 - snapshot_id = result.result["snapshot"]["id"] - - result = yield client.list_snapshots(env_id) - assert result.code == 200 - assert len(result.result["snapshots"]) == 1 - assert result.result["snapshots"][0]["id"] == snapshot_id - - while result.result["snapshots"][0]["finished"] is None: - result = yield client.list_snapshots(env_id) - assert result.code == 200 - yield gen.sleep(0.1) - - # Change the value of the resource - resource_container.Provider.set("agent1", "key", "other") - - # try to do a restore - result = yield client.restore_snapshot(env_id, snapshot_id) - assert result.code == 200 - restore_id = result.result["restore"]["id"] - - result = yield client.list_restores(env_id) - assert result.code == 200 - assert len(result.result["restores"]) == 1 - - result = yield client.get_restore_status(env_id, restore_id) - assert result.code == 200 - while result.result["restore"]["finished"] is None: - result = yield client.get_restore_status(env_id, restore_id) - assert result.code == 200 - yield gen.sleep(0.1) - - assert resource_container.Provider.get("agent1", "key") == "value" - - # get a snapshot - result = yield client.get_snapshot(env_id, snapshot_id) - assert result.code == 200 - assert result.result["snapshot"]["id"] == snapshot_id - - # delete the restore - result = yield client.delete_restore(env_id, restore_id) - assert result.code == 200 - - # delete the snapshot - result = yield client.delete_snapshot(env_id, snapshot_id) - assert result.code == 200 - - @pytest.mark.gen_test def test_server_agent_api(resource_container, client, server, io_loop): agentmanager = server.get_endpoint(SLICE_AGENT_MANAGER) @@ -1216,9 +1032,6 @@ def test_server_agent_api(resource_container, client, server, io_loop): 'requires': [], 'purged': False, 'send_event': False, - 'state_id': '', - 'allow_restore': True, - 'allow_snapshot': True, }, {'key': 'key2', 'value': 'value', @@ -1226,9 +1039,6 @@ def test_server_agent_api(resource_container, client, server, io_loop): 'requires': [], 'purged': False, 'send_event': False, - 'state_id': '', - 'allow_restore': True, - 'allow_snapshot': True, }] result = yield client.put_version(tid=env_id, version=version, resources=resources, unknowns=[], version_info={}) @@ -1278,9 +1088,6 @@ def test_get_facts(resource_container, client, server, io_loop): 'requires': [], 'purged': False, 'send_event': False, - 'state_id': '', - 'allow_restore': True, - 'allow_snapshot': True, }] result = yield client.put_version(tid=env_id, version=version, resources=resources, unknowns=[], version_info={}) @@ -1325,9 +1132,6 @@ def test_purged_facts(resource_container, client, server, io_loop, environment): 'requires': [], 'purged': False, 'send_event': False, - 'state_id': '', - 'allow_restore': True, - 'allow_snapshot': True, }] result = yield client.put_version(tid=environment, version=version, resources=resources, unknowns=[], version_info={}) @@ -1574,9 +1378,6 @@ def test_unkown_parameters(resource_container, client, server, io_loop): 'requires': [], 'purged': False, 'send_event': False, - 'state_id': '', - 'allow_restore': True, - 'allow_snapshot': True, }] unknowns = [{"resource": resource_id_wov, "parameter": "length", "source": "fact"}] @@ -1627,9 +1428,6 @@ def test_fail(resource_container, client, server, io_loop): 'requires': [], 'purged': False, 'send_event': False, - 'state_id': '', - 'allow_restore': True, - 'allow_snapshot': True, }, {'key': 'key2', 'value': 'value', @@ -1637,9 +1435,6 @@ def test_fail(resource_container, client, server, io_loop): 'requires': ['test::Fail[agent1,key=key],v=%d' % version], 'purged': False, 'send_event': False, - 'state_id': '', - 'allow_restore': True, - 'allow_snapshot': True, }, {'key': 'key3', 'value': 'value', @@ -1647,9 +1442,6 @@ def test_fail(resource_container, client, server, io_loop): 'requires': ['test::Fail[agent1,key=key],v=%d' % version], 'purged': False, 'send_event': False, - 'state_id': '', - 'allow_restore': True, - 'allow_snapshot': True, }, {'key': 'key4', 'value': 'value', @@ -1657,9 +1449,6 @@ def test_fail(resource_container, client, server, io_loop): 'requires': ['test::Resource[agent1,key=key3],v=%d' % version], 'purged': False, 'send_event': False, - 'state_id': '', - 'allow_restore': True, - 'allow_snapshot': True, }, {'key': 'key5', 'value': 'value', @@ -1668,9 +1457,6 @@ def test_fail(resource_container, client, server, io_loop): 'test::Fail[agent1,key=key],v=%d' % version], 'purged': False, 'send_event': False, - 'state_id': '', - 'allow_restore': True, - 'allow_snapshot': True, }] result = yield client.put_version(tid=env_id, version=version, resources=resources, unknowns=[], version_info={}) @@ -1739,9 +1525,6 @@ def make_version(offset=0): 'requires': [], 'purged': False, 'send_event': False, - 'state_id': '', - 'allow_restore': True, - 'allow_snapshot': True, }, {'key': 'key2', 'value': 'value', @@ -1749,9 +1532,6 @@ def make_version(offset=0): 'requires': ['test::Wait[agent1,key=key],v=%d' % version], 'purged': False, 'send_event': False, - 'state_id': '', - 'allow_restore': True, - 'allow_snapshot': True, }, {'key': 'key3', 'value': 'value', @@ -1759,9 +1539,6 @@ def make_version(offset=0): 'requires': [], 'purged': False, 'send_event': False, - 'state_id': '', - 'allow_restore': True, - 'allow_snapshot': True, }, {'key': 'key4', 'value': 'value', @@ -1769,9 +1546,6 @@ def make_version(offset=0): 'requires': ['test::Resource[agent1,key=key3],v=%d' % version], 'purged': False, 'send_event': False, - 'state_id': '', - 'allow_restore': True, - 'allow_snapshot': True, }, {'key': 'key5', 'value': 'value', @@ -1780,9 +1554,6 @@ def make_version(offset=0): 'test::Wait[agent1,key=key],v=%d' % version], 'purged': False, 'send_event': False, - 'state_id': '', - 'allow_restore': True, - 'allow_snapshot': True, }] return version, resources @@ -1893,9 +1664,6 @@ def make_version(offset=0): 'requires': ['test::Resource[%s,key=key3],v=%d' % (agent, version)], 'purged': False, 'send_event': False, - 'state_id': '', - 'allow_restore': True, - 'allow_snapshot': True, }, {'key': 'key2', 'value': 'value', @@ -1903,9 +1671,6 @@ def make_version(offset=0): 'requires': ['test::Wait[%s,key=key],v=%d' % (agent, version)], 'purged': False, 'send_event': False, - 'state_id': '', - 'allow_restore': True, - 'allow_snapshot': True, }, {'key': 'key3', 'value': 'value', @@ -1913,9 +1678,6 @@ def make_version(offset=0): 'requires': [], 'purged': False, 'send_event': False, - 'state_id': '', - 'allow_restore': True, - 'allow_snapshot': True, }, {'key': 'key4', 'value': 'value', @@ -1923,9 +1685,6 @@ def make_version(offset=0): 'requires': ['test::Resource[%s,key=key3],v=%d' % (agent, version)], 'purged': False, 'send_event': False, - 'state_id': '', - 'allow_restore': True, - 'allow_snapshot': True, }, {'key': 'key5', 'value': 'value', @@ -1934,9 +1693,6 @@ def make_version(offset=0): 'test::Wait[%s,key=key],v=%d' % (agent, version)], 'purged': False, 'send_event': False, - 'state_id': '', - 'allow_restore': True, - 'allow_snapshot': True, }]) return version, resources @@ -2019,10 +1775,7 @@ def test_cross_agent_deps(resource_container, io_loop, server, client): 'value': 'value1', 'id': 'test::Resource[agent1,key=key1],v=%d' % version, 'purged': False, - 'state_id': '', 'send_event': False, - 'allow_restore': True, - 'allow_snapshot': True, 'requires': ['test::Wait[agent1,key=key2],v=%d' % version, 'test::Resource[agent2,key=key3],v=%d' % version], }, {'key': 'key2', @@ -2030,30 +1783,21 @@ def test_cross_agent_deps(resource_container, io_loop, server, client): 'id': 'test::Wait[agent1,key=key2],v=%d' % version, 'requires': [], 'purged': False, - 'state_id': '', 'send_event': False, - 'allow_restore': True, - 'allow_snapshot': True, }, {'key': 'key3', 'value': 'value3', 'id': 'test::Resource[agent2,key=key3],v=%d' % version, 'requires': [], 'purged': False, - 'state_id': '', 'send_event': False, - 'allow_restore': True, - 'allow_snapshot': True, }, {'key': 'key4', 'value': 'value4', 'id': 'test::Resource[agent2,key=key4],v=%d' % version, 'requires': [], 'purged': False, - 'state_id': '', 'send_event': False, - 'allow_restore': True, - 'allow_snapshot': True, } ] @@ -2117,10 +1861,7 @@ def test_dryrun_scale(resource_container, io_loop, server, client): 'value': 'value%d' % i, 'id': 'test::Resource[agent1,key=key%d],v=%d' % (i, version), 'purged': False, - 'state_id': '', 'send_event': False, - 'allow_restore': True, - 'allow_snapshot': True, 'requires': [], }) @@ -2176,9 +1917,6 @@ def test_dryrun_failures(resource_container, io_loop, server, client): 'id': 'test::Noprov[agent1,key=key1],v=%d' % version, 'purged': False, 'send_event': False, - 'state_id': '', - 'allow_restore': True, - 'allow_snapshot': True, 'requires': [], }, {'key': 'key2', @@ -2186,9 +1924,6 @@ def test_dryrun_failures(resource_container, io_loop, server, client): 'id': 'test::FailFast[agent1,key=key2],v=%d' % version, 'purged': False, 'send_event': False, - 'state_id': '', - 'allow_restore': True, - 'allow_snapshot': True, 'requires': [], }, {'key': 'key2', @@ -2196,9 +1931,6 @@ def test_dryrun_failures(resource_container, io_loop, server, client): 'id': 'test::DoesNotExist[agent1,key=key2],v=%d' % version, 'purged': False, 'send_event': False, - 'state_id': '', - 'allow_restore': True, - 'allow_snapshot': True, 'requires': [], } ] @@ -2266,9 +1998,6 @@ def test_send_events(resource_container, io_loop, environment, server, client): 'id': res_id_1, 'send_event': False, 'purged': False, - 'state_id': '', - 'allow_restore': True, - 'allow_snapshot': True, 'requires': ['test::Resource[agent1,key=key2],v=%d' % version], }, {'key': 'key2', @@ -2277,9 +2006,6 @@ def test_send_events(resource_container, io_loop, environment, server, client): 'send_event': True, 'requires': [], 'purged': False, - 'state_id': '', - 'allow_restore': True, - 'allow_snapshot': True, } ] @@ -2336,9 +2062,6 @@ def test_send_events_cross_agent(resource_container, io_loop, environment, serve 'id': res_id_1, 'send_event': False, 'purged': False, - 'state_id': '', - 'allow_restore': True, - 'allow_snapshot': True, 'requires': ['test::Resource[agent2,key=key2],v=%d' % version], }, {'key': 'key2', @@ -2347,9 +2070,6 @@ def test_send_events_cross_agent(resource_container, io_loop, environment, serve 'send_event': True, 'requires': [], 'purged': False, - 'state_id': '', - 'allow_restore': True, - 'allow_snapshot': True, } ] @@ -2404,9 +2124,6 @@ def test_send_events_cross_agent_restart(resource_container, io_loop, environmen 'id': res_id_1, 'send_event': False, 'purged': False, - 'state_id': '', - 'allow_restore': True, - 'allow_snapshot': True, 'requires': ['test::Resource[agent2,key=key2],v=%d' % version], }, {'key': 'key2', @@ -2415,9 +2132,6 @@ def test_send_events_cross_agent_restart(resource_container, io_loop, environmen 'send_event': True, 'requires': [], 'purged': False, - 'state_id': '', - 'allow_restore': True, - 'allow_snapshot': True, } ] @@ -2488,9 +2202,6 @@ def test_auto_deploy(io_loop, server, client, resource_container, environment): 'id': 'test::Resource[agent1,key=key1],v=%d' % version, 'send_event': False, 'purged': False, - 'state_id': '', - 'allow_restore': True, - 'allow_snapshot': True, 'requires': ['test::Resource[agent1,key=key2],v=%d' % version], }, {'key': 'key2', @@ -2499,9 +2210,6 @@ def test_auto_deploy(io_loop, server, client, resource_container, environment): 'send_event': False, 'requires': [], 'purged': False, - 'state_id': '', - 'allow_restore': True, - 'allow_snapshot': True, }, {'key': 'key3', 'value': None, @@ -2509,9 +2217,6 @@ def test_auto_deploy(io_loop, server, client, resource_container, environment): 'send_event': False, 'requires': [], 'purged': True, - 'state_id': '', - 'allow_restore': True, - 'allow_snapshot': True, } ] @@ -2562,9 +2267,6 @@ def test_auto_deploy_no_splay(io_loop, server, client, resource_container, envir 'id': 'test::Resource[agent1,key=key1],v=%d' % version, 'send_event': False, 'purged': False, - 'state_id': '', - 'allow_restore': True, - 'allow_snapshot': True, 'requires': ['test::Resource[agent1,key=key2],v=%d' % version], }, ] @@ -2620,9 +2322,6 @@ def test_autostart_mapping(io_loop, server, client, resource_container, environm 'id': 'test::Resource[agent1,key=key1],v=%d' % version, 'send_event': False, 'purged': False, - 'state_id': '', - 'allow_restore': True, - 'allow_snapshot': True, 'requires': [], }, {'key': 'key1', @@ -2630,9 +2329,6 @@ def test_autostart_mapping(io_loop, server, client, resource_container, environm 'id': 'test::Resource[agent2,key=key1],v=%d' % version, 'send_event': False, 'purged': False, - 'state_id': '', - 'allow_restore': True, - 'allow_snapshot': True, 'requires': [], }, ] @@ -2687,9 +2383,6 @@ def test_autostart_clear_environment(io_loop, server_multi, client_multi, resour 'id': 'test::Resource[agent1,key=key1],v=%d' % version, 'send_event': False, 'purged': False, - 'state_id': '', - 'allow_restore': True, - 'allow_snapshot': True, 'requires': [], } ] @@ -2741,9 +2434,6 @@ def test_autostart_clear_environment(io_loop, server_multi, client_multi, resour 'id': 'test::Resource[agent1,key=key1],v=%d' % version, 'send_event': False, 'purged': False, - 'state_id': '', - 'allow_restore': True, - 'allow_snapshot': True, 'requires': [], } ] @@ -2884,9 +2574,6 @@ def get_resource(self, 'id': 'test::Resource[%s,key=%s],v=%d' % (agent, key, version), 'send_event': True, 'purged': False, - 'state_id': '', - 'allow_restore': True, - 'allow_snapshot': True, 'requires': requires, } @@ -2985,9 +2672,6 @@ def test_deploy_and_events(io_loop, client, server, environment, resource_contai 'id': 'test::Resource[agent1,key=key1],v=%d' % version, 'send_event': True, 'purged': False, - 'state_id': '', - 'allow_restore': True, - 'allow_snapshot': True, 'requires': [], }, dep, @@ -3047,9 +2731,6 @@ def test_deploy_and_events_failed(io_loop, client, server, environment, resource 'id': 'test::Resource[agent1,key=key1],v=%d' % version, 'send_event': True, 'purged': False, - 'state_id': '', - 'allow_restore': True, - 'allow_snapshot': True, 'requires': [], }, {'key': 'key2', @@ -3057,9 +2738,6 @@ def test_deploy_and_events_failed(io_loop, client, server, environment, resource 'id': 'test::BadEvents[agent1,key=key2],v=%d' % version, 'send_event': True, 'purged': False, - 'state_id': '', - 'allow_restore': True, - 'allow_snapshot': True, 'requires': ['test::Resource[agent1,key=key1],v=%d' % version], }, ] @@ -3116,9 +2794,6 @@ def test_reload(io_loop, client, server, environment, resource_container, dep_st 'id': 'test::Resource[agent1,key=key2],v=%d' % version, 'send_event': True, 'purged': False, - 'state_id': '', - 'allow_restore': True, - 'allow_snapshot': True, 'requires': ['test::Resource[agent1,key=key1],v=%d' % version], }, dep