diff --git a/.gitignore b/.gitignore index 7811066e..37679273 100644 --- a/.gitignore +++ b/.gitignore @@ -62,3 +62,4 @@ examples/Python_Script/databases/ # Ignore local changes as they happen with every execution. If something changes, the commit must be forced. docs/notebooks/data/prepared_dbs/demo_poc.sqlite +conflowgen/data/tools/ diff --git a/conflowgen/api/database_chooser.py b/conflowgen/api/database_chooser.py index 2da4817e..a64a601e 100644 --- a/conflowgen/api/database_chooser.py +++ b/conflowgen/api/database_chooser.py @@ -45,8 +45,8 @@ def load_existing_sqlite_database(self, file_name: str) -> None: """ if self.peewee_sqlite_db is not None: self._close_and_reset_db() - self.peewee_sqlite_db = self.sqlite_database_connection.choose_database(file_name, create=False, reset=False) DataSummariesCache.reset_cache() + self.peewee_sqlite_db = self.sqlite_database_connection.choose_database(file_name, create=False, reset=False) def create_new_sqlite_database( self, @@ -91,7 +91,8 @@ def close_current_connection(self) -> None: raise NoCurrentConnectionException("You must first create a connection to an SQLite database.") def _close_and_reset_db(self): - self.logger.debug("Closing current database connection.") + path_to_sqlite_database = self.sqlite_database_connection.path_to_sqlite_database + self.logger.debug(f"Closing current database connection {path_to_sqlite_database}.") self.peewee_sqlite_db.close() self.peewee_sqlite_db = None DataSummariesCache.reset_cache() diff --git a/conflowgen/application/models/random_seed_store.py b/conflowgen/application/models/random_seed_store.py new file mode 100644 index 00000000..41d022ca --- /dev/null +++ b/conflowgen/application/models/random_seed_store.py @@ -0,0 +1,22 @@ +from peewee import AutoField, CharField, IntegerField, BooleanField + +from conflowgen.domain_models.base_model import BaseModel + + +class RandomSeedStore(BaseModel): + """ + This table contains a random seed for each class or function that contains randomness + """ + id = AutoField() + + name = CharField( + help_text="The name of the class, function, or other type of object." + ) + + is_random = BooleanField( + help_text="Whether the value is meant to change between invocations of the generation process." + ) + + random_seed = IntegerField( + help_text="The last used random seed." + ) diff --git a/conflowgen/application/repositories/random_seed_store_repository.py b/conflowgen/application/repositories/random_seed_store_repository.py new file mode 100644 index 00000000..21e76e97 --- /dev/null +++ b/conflowgen/application/repositories/random_seed_store_repository.py @@ -0,0 +1,76 @@ +import logging +import random +import typing +import time + +from conflowgen.application.models.random_seed_store import RandomSeedStore + + +class RandomSeedStoreRepository: + + def __init__(self): + self.logger = logging.getLogger("conflowgen") + + def get_random_seed(self, seed_name: str, log_loading_process: bool = False) -> float: + random_seed: float + random_seed_store = RandomSeedStore.get_or_none( + RandomSeedStore.name == seed_name + ) + if random_seed_store is not None: + if random_seed_store.is_random: + # there is a previous seed but we are told to overwrite it + previous_seed = random_seed_store.random_seed + random_seed = self._get_random_seed() + random_seed_store.random_seed = random_seed + random_seed_store.save() + if log_loading_process: + self.logger.debug(f"Overwrite seed {previous_seed} with {random_seed} for '{seed_name}'") + else: + # there is a previous seed and we should re-use it + random_seed = random_seed_store.random_seed + if log_loading_process: + self.logger.debug(f"Re-use seed {random_seed} for '{seed_name}'") + else: + # there is no previous seed available, enter the current seed and return its value + random_seed = self._get_random_seed() + RandomSeedStore.create( + name=seed_name, + random_seed=random_seed, + is_random=True + ) + if log_loading_process: + self.logger.debug(f"Randomly set seed {random_seed} for '{seed_name}'") + return random_seed + + @staticmethod + def _get_random_seed() -> int: + return int(time.time()) + + def fix_random_seed( + self, seed_name: str, random_seed: typing.Optional[int], log_loading_process: bool = False + ) -> None: + if random_seed is None: + random_seed = self._get_random_seed() + random_seed_store = RandomSeedStore.get_or_none( + RandomSeedStore.name == seed_name + ) + if random_seed_store is None: + random_seed_store = RandomSeedStore.create( + name=seed_name, + is_random=False, + random_seed=random_seed + ) + else: + random_seed_store.random_seed = random_seed + if log_loading_process: + self.logger.debug(f"Set seed {random_seed} for '{seed_name}'") + random_seed_store.save() + + +_random_seed_store_repository = RandomSeedStoreRepository() + + +def get_initialised_random_object(seed_name: str, log_loading_process: bool = True) -> random.Random: + random_seed = RandomSeedStoreRepository().get_random_seed(seed_name, log_loading_process=log_loading_process) + seeded_random = random.Random(x=random_seed) + return seeded_random diff --git a/conflowgen/application/services/average_container_dwell_time_calculator_service.py b/conflowgen/application/services/average_container_dwell_time_calculator_service.py index 708420e8..49b263c9 100644 --- a/conflowgen/application/services/average_container_dwell_time_calculator_service.py +++ b/conflowgen/application/services/average_container_dwell_time_calculator_service.py @@ -14,7 +14,8 @@ class AverageContainerDwellTimeCalculatorService: - def get_average_container_dwell_time(self, start_date: datetime.date, end_date: datetime.date) -> float: + @staticmethod + def get_average_container_dwell_time(start_date: datetime.date, end_date: datetime.date) -> float: inbound_vehicle_capacity = InboundAndOutboundVehicleCapacityCalculatorService.get_inbound_capacity_of_vehicles( start_date=start_date, end_date=end_date diff --git a/conflowgen/database_connection/create_tables.py b/conflowgen/database_connection/create_tables.py index 3a3d70f2..69d6646d 100644 --- a/conflowgen/database_connection/create_tables.py +++ b/conflowgen/database_connection/create_tables.py @@ -3,6 +3,7 @@ import peewee from conflowgen.application.models.container_flow_generation_properties import ContainerFlowGenerationProperties +from conflowgen.application.models.random_seed_store import RandomSeedStore from conflowgen.domain_models.arrival_information import TruckArrivalInformationForPickup, \ TruckArrivalInformationForDelivery from conflowgen.domain_models.container import Container @@ -40,7 +41,8 @@ def create_tables(sql_db_connection: peewee.Database) -> peewee.Database: TruckArrivalInformationForPickup, TruckArrivalInformationForDelivery, StorageRequirementDistribution, - ContainerDwellTimeDistribution + ContainerDwellTimeDistribution, + RandomSeedStore, ]) for table_with_index in ( Destination, diff --git a/conflowgen/database_connection/sqlite_database_connection.py b/conflowgen/database_connection/sqlite_database_connection.py index f0cd3138..618c5d62 100644 --- a/conflowgen/database_connection/sqlite_database_connection.py +++ b/conflowgen/database_connection/sqlite_database_connection.py @@ -57,6 +57,7 @@ def __init__(self, sqlite_databases_directory: Optional[str] = None): sqlite_databases_directory = self.SQLITE_DEFAULT_DIR sqlite_databases_directory = os.path.abspath(sqlite_databases_directory) self.sqlite_databases_directory = sqlite_databases_directory + self.path_to_sqlite_database = "" self.logger = logging.getLogger("conflowgen") @@ -82,16 +83,15 @@ def choose_database( **seeder_options ) -> SqliteDatabase: if database_name == ":memory:": - path_to_sqlite_database = ":memory:" + self.path_to_sqlite_database = ":memory:" sqlite_database_existed_before = False else: - path_to_sqlite_database, sqlite_database_existed_before = self._load_or_create_sqlite_file_on_hard_drive( - database_name=database_name, create=create, reset=reset - ) + self.path_to_sqlite_database, sqlite_database_existed_before = ( + self._load_or_create_sqlite_file_on_hard_drive(database_name=database_name, create=create, reset=reset)) - self.logger.debug(f"Opening file {path_to_sqlite_database}") + self.logger.debug(f"Opening file {self.path_to_sqlite_database}") self.sqlite_db_connection = SqliteDatabase( - path_to_sqlite_database, + self.path_to_sqlite_database, pragmas=self.SQLITE_DEFAULT_SETTINGS ) database_proxy.initialize(self.sqlite_db_connection) @@ -103,12 +103,12 @@ def choose_database( self.logger.debug(f'foreign_keys: {self.sqlite_db_connection.foreign_keys}') if not sqlite_database_existed_before or reset: - self.logger.debug(f"Creating new database at {path_to_sqlite_database}") + self.logger.debug(f"Creating new database at {self.path_to_sqlite_database}") create_tables(self.sqlite_db_connection) self.logger.debug("Seed with default values...") seed_all_distributions(**seeder_options) else: - self.logger.debug(f"Open existing database at {path_to_sqlite_database}") + self.logger.debug(f"Open existing database at {self.path_to_sqlite_database}") container_flow_properties: ContainerFlowGenerationProperties | None = \ ContainerFlowGenerationProperties.get_or_none() diff --git a/conflowgen/domain_models/factories/container_factory.py b/conflowgen/domain_models/factories/container_factory.py index 4b35cff2..bd3aab8a 100644 --- a/conflowgen/domain_models/factories/container_factory.py +++ b/conflowgen/domain_models/factories/container_factory.py @@ -1,7 +1,6 @@ from __future__ import annotations import math -import random from typing import Dict, MutableSequence, Sequence, Type from conflowgen.domain_models.container import Container @@ -19,6 +18,7 @@ from conflowgen.domain_models.repositories.large_scheduled_vehicle_repository import LargeScheduledVehicleRepository from conflowgen.domain_models.vehicle import AbstractLargeScheduledVehicle, LargeScheduledVehicle from conflowgen.tools.distribution_approximator import DistributionApproximator +from conflowgen.application.repositories.random_seed_store_repository import get_initialised_random_object class ContainerFactory: @@ -26,12 +26,10 @@ class ContainerFactory: Creates containers according to the distributions which are either hard-coded or stored in the database. """ - ignored_capacity = ContainerLength.get_teu_factor(ContainerLength.other) - - random_seed = 1 + ignored_capacity = ContainerLength.get_maximum_teu_factor() def __init__(self): - self.seeded_random = random.Random(x=self.random_seed) + self.seeded_random = get_initialised_random_object(self.__class__.__name__) self.mode_of_transportation_distribution: dict[ModeOfTransport, dict[ModeOfTransport, float]] | None = None self.container_length_distribution: dict[ContainerLength, float] | None = None self.container_weight_distribution: dict[ContainerLength, dict[int, float]] | None = None diff --git a/conflowgen/flow_generator/abstract_truck_for_containers_manager.py b/conflowgen/flow_generator/abstract_truck_for_containers_manager.py index 1d2483e1..c52d450d 100644 --- a/conflowgen/flow_generator/abstract_truck_for_containers_manager.py +++ b/conflowgen/flow_generator/abstract_truck_for_containers_manager.py @@ -2,10 +2,10 @@ import abc import logging import math -import random -from typing import List, Tuple, Union, Optional, Dict, Sequence +import typing from conflowgen.tools.weekly_distribution import WeeklyDistribution +from ..application.repositories.random_seed_store_repository import get_initialised_random_object from ..domain_models.data_types.storage_requirement import StorageRequirement from ..domain_models.container import Container from ..domain_models.distribution_repositories.container_dwell_time_distribution_repository import \ @@ -29,15 +29,18 @@ class AbstractTruckForContainersManager(abc.ABC): def __init__(self): self.logger = logging.getLogger("conflowgen") + self.seeded_random = get_initialised_random_object(self.__class__.__name__) + self.container_dwell_time_distribution_repository = ContainerDwellTimeDistributionRepository() self.container_dwell_time_distributions: \ - Dict[ModeOfTransport, Dict[ModeOfTransport, Dict[StorageRequirement, ContinuousDistribution]]] | None \ + typing.Dict[ModeOfTransport, typing.Dict[ + ModeOfTransport, typing.Dict[StorageRequirement, ContinuousDistribution]]] | None \ = None self.truck_arrival_distribution_repository = TruckArrivalDistributionRepository() self.truck_arrival_distributions: \ - Dict[ModeOfTransport, Dict[StorageRequirement, WeeklyDistribution | None]] = { + typing.Dict[ModeOfTransport, typing.Dict[StorageRequirement, WeeklyDistribution | None]] = { vehicle: { storage_requirement: None for storage_requirement in StorageRequirement @@ -45,7 +48,7 @@ def __init__(self): } self.vehicle_factory = VehicleFactory() - self.time_window_length_in_hours: Optional[int] = None + self.time_window_length_in_hours: typing.Optional[int] = None @abc.abstractmethod def _get_container_dwell_time_distribution( @@ -64,7 +67,7 @@ def reload_distributions( self ) -> None: # noinspection PyTypeChecker - hour_of_the_week_fraction_pairs: List[Union[Tuple[int, float], Tuple[int, int]]] = \ + hour_of_the_week_fraction_pairs: typing.List[typing.Union[typing.Tuple[int, float], typing.Tuple[int, int]]] = \ list(self.truck_arrival_distribution_repository.get_distribution().items()) self.time_window_length_in_hours = hour_of_the_week_fraction_pairs[1][0] - hour_of_the_week_fraction_pairs[0][0] @@ -73,7 +76,7 @@ def reload_distributions( def _update_truck_arrival_and_container_dwell_time_distributions( self, - hour_of_the_week_fraction_pairs: List[Union[Tuple[int, float], Tuple[int, int]]] + hour_of_the_week_fraction_pairs: typing.List[typing.Union[typing.Tuple[int, float], typing.Tuple[int, int]]] ) -> None: for vehicle_type in ModeOfTransport: for storage_requirement in StorageRequirement: @@ -111,14 +114,15 @@ def _get_distributions( return container_dwell_time_distribution, truck_arrival_distribution @abc.abstractmethod - def _get_truck_arrival_distributions(self, container: Container) -> Dict[StorageRequirement, WeeklyDistribution]: + def _get_truck_arrival_distributions(self, container: Container) -> typing.Dict[ + StorageRequirement, WeeklyDistribution]: pass def _get_time_window_of_truck_arrival( self, container_dwell_time_distribution: ContinuousDistribution, - truck_arrival_distribution_slice: Dict[int, float], - _debug_check_distribution_property: Optional[str] = None + truck_arrival_distribution_slice: typing.Dict[int, float], + _debug_check_distribution_property: typing.Optional[str] = None ) -> int: """ Returns: @@ -152,7 +156,7 @@ def _get_time_window_of_truck_arrival( else: raise UnknownDistributionPropertyException(_debug_check_distribution_property) else: - selected_time_window = random.choices( + selected_time_window = self.seeded_random.choices( population=time_windows_for_truck_arrival, weights=total_probabilities )[0] @@ -178,7 +182,7 @@ def _get_time_window_of_truck_arrival( return selected_time_window @staticmethod - def _drop_where_zero(sequence: Sequence, filter_sequence: Sequence) -> list: + def _drop_where_zero(sequence: typing.Sequence, filter_sequence: typing.Sequence) -> list: new_sequence = [] for element, filter_element in zip(sequence, filter_sequence): if filter_element: diff --git a/conflowgen/flow_generator/allocate_space_for_containers_delivered_by_truck_service.py b/conflowgen/flow_generator/allocate_space_for_containers_delivered_by_truck_service.py index 322e4dc7..28bbdba4 100644 --- a/conflowgen/flow_generator/allocate_space_for_containers_delivered_by_truck_service.py +++ b/conflowgen/flow_generator/allocate_space_for_containers_delivered_by_truck_service.py @@ -1,8 +1,8 @@ from __future__ import annotations import logging -import random from typing import Dict, Type, List +from conflowgen.application.repositories.random_seed_store_repository import get_initialised_random_object from conflowgen.domain_models.container import Container from conflowgen.domain_models.distribution_repositories.mode_of_transport_distribution_repository import \ ModeOfTransportDistributionRepository @@ -18,6 +18,8 @@ class AllocateSpaceForContainersDeliveredByTruckService: ignored_capacity = ContainerLength.get_teu_factor(ContainerLength.other) def __init__(self): + self.seeded_random = get_initialised_random_object(self.__class__.__name__) + self.logger = logging.getLogger("conflowgen") self.mode_of_transport_distribution_repository = ModeOfTransportDistributionRepository() self.mode_of_transport_distribution: Dict[ModeOfTransport, Dict[ModeOfTransport, float]] | None = None @@ -155,7 +157,7 @@ def _pick_vehicle_type( return None # pick vehicle type - vehicle_type: ModeOfTransport = random.choices( + vehicle_type: ModeOfTransport = self.seeded_random.choices( population=vehicle_types, weights=frequency_of_vehicle_types )[0] @@ -178,7 +180,7 @@ def _pick_vehicle( "by trucks.") return None - vehicle: Type[AbstractLargeScheduledVehicle] = random.choices( + vehicle: Type[AbstractLargeScheduledVehicle] = self.seeded_random.choices( population=list(vehicle_distribution.keys()), weights=list(vehicle_distribution.values()) )[0] diff --git a/conflowgen/flow_generator/assign_destination_to_container_service.py b/conflowgen/flow_generator/assign_destination_to_container_service.py index ef1f6951..5af2aeab 100644 --- a/conflowgen/flow_generator/assign_destination_to_container_service.py +++ b/conflowgen/flow_generator/assign_destination_to_container_service.py @@ -1,9 +1,9 @@ from __future__ import annotations import logging -import random -from typing import Iterable, Dict +import typing +from conflowgen.application.repositories.random_seed_store_repository import get_initialised_random_object from conflowgen.domain_models.container import Container from conflowgen.domain_models.distribution_repositories.container_destination_distribution_repository import \ ContainerDestinationDistributionRepository @@ -16,8 +16,10 @@ class AssignDestinationToContainerService: logger = logging.getLogger("conflowgen") def __init__(self): + self.seeded_random = get_initialised_random_object(self.__class__.__name__) + self.repository = ContainerDestinationDistributionRepository() - self.distribution: Dict[Schedule, Dict[Destination, float]] | None = None + self.distribution: typing.Dict[Schedule, typing.Dict[Destination, float]] | None = None self.reload_distributions() def reload_distributions(self): @@ -37,7 +39,7 @@ def assign(self) -> None: in the following. This step can only be done if the next destinations of the vehicle are determined in the schedule (this is an optional user input). The frequency is expressed in boxes. """ - destination_with_distinct_schedules: Iterable[Destination] = Destination.select( + destination_with_distinct_schedules: typing.Iterable[Destination] = Destination.select( Destination.belongs_to_schedule).distinct() schedules = [ destination.belongs_to_schedule @@ -49,7 +51,7 @@ def assign(self) -> None: self.logger.debug(f"Assign destinations to containers that leave the terminal with the service " f"'{schedule.service_name}' of the vehicle type {schedule.vehicle_type}, " f"progress: {i+1} / {number_iterations} ({100*(i + 1)/number_iterations:.2f}%)") - containers_moving_according_to_schedule: Iterable[Container] = Container.select().join( + containers_moving_according_to_schedule: typing.Iterable[Container] = Container.select().join( LargeScheduledVehicle, on=Container.picked_up_by_large_scheduled_vehicle ).where( Container.picked_up_by_large_scheduled_vehicle.schedule == schedule @@ -60,7 +62,7 @@ def assign(self) -> None: container: Container for container in containers_moving_according_to_schedule: - sampled_destination = random.choices( + sampled_destination = self.seeded_random.choices( population=destinations, weights=frequency_of_destinations )[0] diff --git a/conflowgen/flow_generator/large_scheduled_vehicle_for_onward_transportation_manager.py b/conflowgen/flow_generator/large_scheduled_vehicle_for_onward_transportation_manager.py index b18b995c..402a3dbc 100644 --- a/conflowgen/flow_generator/large_scheduled_vehicle_for_onward_transportation_manager.py +++ b/conflowgen/flow_generator/large_scheduled_vehicle_for_onward_transportation_manager.py @@ -2,7 +2,6 @@ import datetime import logging import math -import random from typing import Tuple, List, Dict, Type, Sequence import numpy as np @@ -10,6 +9,7 @@ from peewee import fn, JOIN, ModelSelect from conflowgen.data_summaries.data_summaries_cache import DataSummariesCache +from ..application.repositories.random_seed_store_repository import get_initialised_random_object from ..domain_models.data_types.container_length import ContainerLength from ..domain_models.data_types.storage_requirement import StorageRequirement from ..domain_models.arrival_information import TruckArrivalInformationForDelivery @@ -25,10 +25,10 @@ class LargeScheduledVehicleForOnwardTransportationManager: - random_seed = 1 def __init__(self): - self.seeded_random = random.Random(x=self.random_seed) + self.seeded_random = get_initialised_random_object(self.__class__.__name__) + self.logger = logging.getLogger("conflowgen") self.schedule_repository = ScheduleRepository() self.large_scheduled_vehicle_repository = self.schedule_repository.large_scheduled_vehicle_repository diff --git a/conflowgen/flow_generator/truck_for_export_containers_manager.py b/conflowgen/flow_generator/truck_for_export_containers_manager.py index 2d89aab6..c4e34e3f 100644 --- a/conflowgen/flow_generator/truck_for_export_containers_manager.py +++ b/conflowgen/flow_generator/truck_for_export_containers_manager.py @@ -1,6 +1,5 @@ from __future__ import annotations import datetime -import random from typing import Dict, Optional from .abstract_truck_for_containers_manager import AbstractTruckForContainersManager, \ @@ -76,7 +75,7 @@ def _get_container_delivery_time( # arrival within the last time slot close_to_time_window_length = self.time_window_length_in_hours - (1 / 60) - random_time_component: float = random.uniform(0, close_to_time_window_length) + random_time_component: float = self.seeded_random.uniform(0, close_to_time_window_length) if _debug_check_distribution_property is not None: if _debug_check_distribution_property == "minimum": diff --git a/conflowgen/flow_generator/truck_for_import_containers_manager.py b/conflowgen/flow_generator/truck_for_import_containers_manager.py index 94d2b0ca..1de0e6cb 100644 --- a/conflowgen/flow_generator/truck_for_import_containers_manager.py +++ b/conflowgen/flow_generator/truck_for_import_containers_manager.py @@ -1,5 +1,4 @@ import datetime -import random from typing import Dict, Optional from .abstract_truck_for_containers_manager import AbstractTruckForContainersManager, \ @@ -58,7 +57,7 @@ def _get_container_pickup_time( # arrival within the last time slot close_to_time_window_length = self.time_window_length_in_hours - (1 / 60) - random_time_component: float = random.uniform(0, close_to_time_window_length) + random_time_component: float = self.seeded_random.uniform(0, close_to_time_window_length) if _debug_check_distribution_property is not None: if _debug_check_distribution_property == "minimum": diff --git a/conflowgen/tests/analyses/test_truck_gate_throughput_analysis_report.py b/conflowgen/tests/analyses/test_truck_gate_throughput_analysis_report.py index 2d04ba11..daaf4f01 100644 --- a/conflowgen/tests/analyses/test_truck_gate_throughput_analysis_report.py +++ b/conflowgen/tests/analyses/test_truck_gate_throughput_analysis_report.py @@ -2,6 +2,7 @@ from conflowgen.analyses.truck_gate_throughput_analysis_report import TruckGateThroughputAnalysisReport from conflowgen.application.models.container_flow_generation_properties import ContainerFlowGenerationProperties +from conflowgen.application.models.random_seed_store import RandomSeedStore from conflowgen.domain_models.arrival_information import TruckArrivalInformationForPickup, \ TruckArrivalInformationForDelivery from conflowgen.domain_models.container import Container @@ -92,7 +93,8 @@ def setUp(self) -> None: ModeOfTransportDistribution, Destination, ContainerFlowGenerationProperties, - Train + Train, + RandomSeedStore ]) mode_of_transport_distribution_seeder.seed() ContainerFlowGenerationProperties.create( diff --git a/conflowgen/tests/api/test_container_flow_generation_manager.py b/conflowgen/tests/api/test_container_flow_generation_manager.py index 592e7403..62c03e3c 100644 --- a/conflowgen/tests/api/test_container_flow_generation_manager.py +++ b/conflowgen/tests/api/test_container_flow_generation_manager.py @@ -4,6 +4,7 @@ from conflowgen.api.container_flow_generation_manager import ContainerFlowGenerationManager from conflowgen.application.models.container_flow_generation_properties import ContainerFlowGenerationProperties +from conflowgen.application.models.random_seed_store import RandomSeedStore from conflowgen.domain_models.distribution_models.mode_of_transport_distribution import ModeOfTransportDistribution from conflowgen.domain_models.distribution_seeders import mode_of_transport_distribution_seeder from conflowgen.domain_models.large_vehicle_schedule import Schedule @@ -18,7 +19,8 @@ def setUp(self) -> None: sqlite_db.create_tables([ ContainerFlowGenerationProperties, ModeOfTransportDistribution, - Schedule + Schedule, + RandomSeedStore ]) mode_of_transport_distribution_seeder.seed() self.container_flow_generation_manager = ContainerFlowGenerationManager() diff --git a/conflowgen/tests/application_models/__init__.py b/conflowgen/tests/application/__init__.py similarity index 100% rename from conflowgen/tests/application_models/__init__.py rename to conflowgen/tests/application/__init__.py diff --git a/conflowgen/tests/application_models/repositories/__init__.py b/conflowgen/tests/application/reports/__init__.py similarity index 100% rename from conflowgen/tests/application_models/repositories/__init__.py rename to conflowgen/tests/application/reports/__init__.py diff --git a/conflowgen/tests/application_models/repositories/test_container_stream_statistics_report.py b/conflowgen/tests/application/reports/test_container_flow_statistics_report.py similarity index 100% rename from conflowgen/tests/application_models/repositories/test_container_stream_statistics_report.py rename to conflowgen/tests/application/reports/test_container_flow_statistics_report.py diff --git a/conflowgen/tests/application/repositories/__init__.py b/conflowgen/tests/application/repositories/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/conflowgen/tests/application_models/repositories/test_container_stream_generation_properties_repository.py b/conflowgen/tests/application/repositories/test_container_flow_generation_properties_repository.py similarity index 100% rename from conflowgen/tests/application_models/repositories/test_container_stream_generation_properties_repository.py rename to conflowgen/tests/application/repositories/test_container_flow_generation_properties_repository.py diff --git a/conflowgen/tests/application/repositories/test_random_seed_store_repository.py b/conflowgen/tests/application/repositories/test_random_seed_store_repository.py new file mode 100644 index 00000000..5f0e0208 --- /dev/null +++ b/conflowgen/tests/application/repositories/test_random_seed_store_repository.py @@ -0,0 +1,63 @@ +import time +import unittest + +from conflowgen.application.models.random_seed_store import RandomSeedStore +from conflowgen.application.repositories.random_seed_store_repository import RandomSeedStoreRepository +from conflowgen.tests.substitute_peewee_database import setup_sqlite_in_memory_db + + +class TestRandomSeedStoreRepository(unittest.TestCase): + + def setUp(self) -> None: + """Create container database in memory""" + self.sqlite_db = setup_sqlite_in_memory_db() + self.sqlite_db.create_tables([ + RandomSeedStore + ]) + self.repository = RandomSeedStoreRepository() + + def test_get_empty_entry(self): + with self.assertLogs('conflowgen', level='DEBUG') as context: + random_seed = self.repository.get_random_seed("empty_entry", True) + self.assertIsInstance(random_seed, int) + self.assertEqual(len(context.output), 1) + logged_message = context.output[0] + self.assertRegex(logged_message, r"Randomly set seed \d+ for 'empty_entry'") + + def test_fix_existing_entry(self): + seed = int(time.time()) + with self.assertLogs('conflowgen', level='DEBUG') as context: + self.repository.fix_random_seed("fix_seed", seed, True) + self.assertEqual(len(context.output), 1) + logged_message = context.output[0] + self.assertRegex(logged_message, r"Set seed \d+ for 'fix_seed'") + + def test_reuse_existing_entry(self): + seed = int(time.time()) + RandomSeedStore.create( + name="reuse_existing", + random_seed=seed, + is_random=True + ) + random_seed = self.repository.get_random_seed("reuse_existing", False) + self.assertEqual(random_seed, seed) + + def test_fix_and_reuse_journey(self): + for _ in range(10): + seed = int(time.time()) + with self.assertLogs('conflowgen', level='DEBUG') as context: + self.repository.fix_random_seed("fix_and_reuse", seed, True) + self.assertEqual(len(context.output), 1) + logged_message = context.output[0] + self.assertRegex(logged_message, rf"Set seed {seed} for 'fix_and_reuse'") + + with self.assertLogs('conflowgen', level='DEBUG') as context: + retrieved_seed = self.repository.get_random_seed("fix_and_reuse", True) + self.assertEqual(len(context.output), 1) + logged_message = context.output[0] + self.assertRegex( + logged_message, + fr"Re-use seed {retrieved_seed} for 'fix_and_reuse'" + ) + + self.assertEqual(seed, retrieved_seed) diff --git a/conflowgen/tests/application/services/__init__.py b/conflowgen/tests/application/services/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/conflowgen/tests/domain_models/factories/test_container_factory__create_for_large_scheduled_vehicle.py b/conflowgen/tests/domain_models/factories/test_container_factory__create_for_large_scheduled_vehicle.py index dfbfaeb0..ccd84951 100644 --- a/conflowgen/tests/domain_models/factories/test_container_factory__create_for_large_scheduled_vehicle.py +++ b/conflowgen/tests/domain_models/factories/test_container_factory__create_for_large_scheduled_vehicle.py @@ -1,6 +1,7 @@ import datetime import unittest +from conflowgen.application.models.random_seed_store import RandomSeedStore from conflowgen.domain_models.container import Container from conflowgen.domain_models.distribution_models.container_length_distribution import ContainerLengthDistribution from conflowgen.domain_models.distribution_models.container_weight_distribution import ContainerWeightDistribution @@ -32,7 +33,8 @@ def setUp(self) -> None: ContainerWeightDistribution, ContainerLengthDistribution, Destination, - StorageRequirementDistribution + StorageRequirementDistribution, + RandomSeedStore ]) mode_of_transport_distribution_seeder.seed() container_weight_distribution_seeder.seed() diff --git a/conflowgen/tests/domain_models/factories/test_container_factory__create_for_truck.py b/conflowgen/tests/domain_models/factories/test_container_factory__create_for_truck.py index 6f1420af..b873d5e0 100644 --- a/conflowgen/tests/domain_models/factories/test_container_factory__create_for_truck.py +++ b/conflowgen/tests/domain_models/factories/test_container_factory__create_for_truck.py @@ -1,6 +1,7 @@ import datetime import unittest +from conflowgen.application.models.random_seed_store import RandomSeedStore from conflowgen.domain_models.arrival_information import TruckArrivalInformationForDelivery, \ TruckArrivalInformationForPickup from conflowgen.domain_models.container import Container @@ -38,7 +39,8 @@ def setUp(self) -> None: TruckArrivalInformationForPickup, ContainerLengthDistribution, Destination, - StorageRequirementDistribution + StorageRequirementDistribution, + RandomSeedStore ]) mode_of_transport_distribution_seeder.seed() container_weight_distribution_seeder.seed() diff --git a/conflowgen/tests/flow_generator/test_allocate_space_for_containers_delivered_by_truck_service.py b/conflowgen/tests/flow_generator/test_allocate_space_for_containers_delivered_by_truck_service.py index 2a6df45e..4e749554 100644 --- a/conflowgen/tests/flow_generator/test_allocate_space_for_containers_delivered_by_truck_service.py +++ b/conflowgen/tests/flow_generator/test_allocate_space_for_containers_delivered_by_truck_service.py @@ -1,6 +1,7 @@ import datetime import unittest +from conflowgen.application.models.random_seed_store import RandomSeedStore from conflowgen.domain_models.arrival_information import TruckArrivalInformationForDelivery, \ TruckArrivalInformationForPickup from conflowgen.domain_models.container import Container @@ -42,7 +43,8 @@ def setUp(self) -> None: ModeOfTransportDistribution, ContainerLengthDistribution, ContainerWeightDistribution, - StorageRequirementDistribution + StorageRequirementDistribution, + RandomSeedStore ]) mode_of_transport_distribution_seeder.seed() diff --git a/conflowgen/tests/flow_generator/test_assign_destination_to_container_service.py b/conflowgen/tests/flow_generator/test_assign_destination_to_container_service.py index 5f7542d2..6818e4c7 100644 --- a/conflowgen/tests/flow_generator/test_assign_destination_to_container_service.py +++ b/conflowgen/tests/flow_generator/test_assign_destination_to_container_service.py @@ -1,6 +1,7 @@ import datetime import unittest +from conflowgen.application.models.random_seed_store import RandomSeedStore from conflowgen.flow_generator.assign_destination_to_container_service import \ AssignDestinationToContainerService from conflowgen.domain_models.arrival_information import TruckArrivalInformationForDelivery, \ @@ -33,7 +34,8 @@ def setUp(self) -> None: Truck, Feeder, DeepSeaVessel, - ModeOfTransportDistribution + ModeOfTransportDistribution, + RandomSeedStore, ]) self.repository = ContainerDestinationDistributionRepository() self.service = AssignDestinationToContainerService() diff --git a/conflowgen/tests/flow_generator/test_container_flow_generator_service__container_flow_data_exists.py b/conflowgen/tests/flow_generator/test_container_flow_generator_service__container_flow_data_exists.py index fa60047f..b6091a1f 100644 --- a/conflowgen/tests/flow_generator/test_container_flow_generator_service__container_flow_data_exists.py +++ b/conflowgen/tests/flow_generator/test_container_flow_generator_service__container_flow_data_exists.py @@ -1,6 +1,7 @@ import unittest from conflowgen import ContainerLength, StorageRequirement +from conflowgen.application.models.random_seed_store import RandomSeedStore from conflowgen.domain_models.container import Container from conflowgen.domain_models.distribution_models.mode_of_transport_distribution import ModeOfTransportDistribution from conflowgen.domain_models.distribution_seeders import mode_of_transport_distribution_seeder @@ -23,7 +24,8 @@ def setUp(self) -> None: Schedule, Destination, Truck, - LargeScheduledVehicle + LargeScheduledVehicle, + RandomSeedStore, ]) mode_of_transport_distribution_seeder.seed() self.container_flow_generator_service = ContainerFlowGenerationService() diff --git a/conflowgen/tests/flow_generator/test_container_flow_generator_service__generate.py b/conflowgen/tests/flow_generator/test_container_flow_generator_service__generate.py index 591fa8c3..1bb2ded2 100644 --- a/conflowgen/tests/flow_generator/test_container_flow_generator_service__generate.py +++ b/conflowgen/tests/flow_generator/test_container_flow_generator_service__generate.py @@ -3,6 +3,7 @@ from conflowgen import PortCallManager from conflowgen.application.models.container_flow_generation_properties import ContainerFlowGenerationProperties +from conflowgen.application.models.random_seed_store import RandomSeedStore from conflowgen.application.repositories.container_flow_generation_properties_repository import \ ContainerFlowGenerationPropertiesRepository from conflowgen.database_connection.create_tables import create_tables @@ -29,7 +30,8 @@ def setUp(self) -> None: ModeOfTransportDistribution, Schedule, StorageRequirementDistribution, - ContainerDwellTimeDistribution + ContainerDwellTimeDistribution, + RandomSeedStore, ]) mode_of_transport_distribution_seeder.seed() container_dwell_time_distribution_seeder.seed() diff --git a/conflowgen/tests/flow_generator/test_distribution_approximator.py b/conflowgen/tests/flow_generator/test_distribution_approximator.py index 65e870c2..01b435ea 100644 --- a/conflowgen/tests/flow_generator/test_distribution_approximator.py +++ b/conflowgen/tests/flow_generator/test_distribution_approximator.py @@ -5,11 +5,20 @@ import collections import unittest +from conflowgen.application.models.random_seed_store import RandomSeedStore from conflowgen.tools.distribution_approximator import DistributionApproximator, SamplerExhaustedException +from conflowgen.tests.substitute_peewee_database import setup_sqlite_in_memory_db class TestDistributionApproximator(unittest.TestCase): + def setUp(self) -> None: + """Create container database in memory""" + sqlite_db = setup_sqlite_in_memory_db() + sqlite_db.create_tables([ + RandomSeedStore + ]) + def test_happy_path(self) -> None: """This is the happy path""" distribution_approximator = DistributionApproximator({ diff --git a/conflowgen/tests/flow_generator/test_large_scheduled_vehicle_for_onward_transportation_manager.py b/conflowgen/tests/flow_generator/test_large_scheduled_vehicle_for_onward_transportation_manager.py index 05bdadb4..55286a7b 100644 --- a/conflowgen/tests/flow_generator/test_large_scheduled_vehicle_for_onward_transportation_manager.py +++ b/conflowgen/tests/flow_generator/test_large_scheduled_vehicle_for_onward_transportation_manager.py @@ -3,6 +3,7 @@ import unittest.mock from typing import Iterable +from conflowgen.application.models.random_seed_store import RandomSeedStore from conflowgen.domain_models.arrival_information import TruckArrivalInformationForDelivery, \ TruckArrivalInformationForPickup from conflowgen.domain_models.container import Container @@ -27,7 +28,7 @@ class TestLargeScheduledVehicleForExportContainersManager(unittest.TestCase): def setUp(self) -> None: """Create container database in memory""" sqlite_db = setup_sqlite_in_memory_db() - sqlite_db.create_tables([ + sqlite_db.create_tables({ Schedule, LargeScheduledVehicle, Train, @@ -41,7 +42,8 @@ def setUp(self) -> None: Destination, ModeOfTransportDistribution, ContainerDwellTimeDistribution, - ]) + RandomSeedStore, + }) mode_of_transport_distribution_seeder.seed() container_dwell_time_distribution_seeder.seed() diff --git a/conflowgen/tests/flow_generator/test_truck_for_export_containers_manager.py b/conflowgen/tests/flow_generator/test_truck_for_export_containers_manager.py index fe96b980..4bb684ce 100644 --- a/conflowgen/tests/flow_generator/test_truck_for_export_containers_manager.py +++ b/conflowgen/tests/flow_generator/test_truck_for_export_containers_manager.py @@ -11,6 +11,7 @@ import matplotlib.pyplot as plt import numpy as np +from conflowgen.application.models.random_seed_store import RandomSeedStore from conflowgen.domain_models.arrival_information import TruckArrivalInformationForDelivery from conflowgen.flow_generator.truck_for_export_containers_manager import \ TruckForExportContainersManager @@ -46,7 +47,8 @@ def setUp(self) -> None: Truck, LargeScheduledVehicle, Schedule, - TruckArrivalInformationForDelivery + TruckArrivalInformationForDelivery, + RandomSeedStore, ]) truck_arrival_distribution_seeder.seed() container_dwell_time_distribution_seeder.seed() @@ -362,9 +364,9 @@ def test_delivery_time_maximum(self): maximum = datetime.datetime(2021, 8, 8, 12) - datetime.timedelta(hours=467) self.assertEqual(maximum, delivery_time) - containder_dwell_time = (container_departure_time - maximum).total_seconds() / 3600 - self.assertGreater(distribution_1.maximum, containder_dwell_time) - self.assertLess(distribution_1.minimum, containder_dwell_time) + container_dwell_time = (container_departure_time - maximum).total_seconds() / 3600 + self.assertGreater(distribution_1.maximum, container_dwell_time) + self.assertLess(distribution_1.minimum, container_dwell_time) def test_delivery_time_average(self): container_departure_time = datetime.datetime( @@ -389,7 +391,7 @@ def test_delivery_time_average(self): self.assertEqual(156, distribution.average) self.assertEqual(468, distribution.maximum) - # the distribution is inversed and the random_time_component is set to zero. Actually, the value can rise close + # the distribution is inverted and the random_time_component is set to zero. Actually, the value can rise close # to one, meaning that the last `-1` would be much smaller. average = datetime.datetime(2021, 8, 8, 12) - datetime.timedelta(hours=(468 - 156 - 1)) self.assertEqual(average, delivery_time) diff --git a/conflowgen/tests/flow_generator/test_truck_for_import_containers_manager.py b/conflowgen/tests/flow_generator/test_truck_for_import_containers_manager.py index 79e668d0..515cbf47 100644 --- a/conflowgen/tests/flow_generator/test_truck_for_import_containers_manager.py +++ b/conflowgen/tests/flow_generator/test_truck_for_import_containers_manager.py @@ -11,6 +11,7 @@ import matplotlib.pyplot as plt import numpy as np +from conflowgen.application.models.random_seed_store import RandomSeedStore from conflowgen.domain_models.data_types.mode_of_transport import ModeOfTransport from conflowgen.domain_models.data_types.storage_requirement import StorageRequirement from conflowgen.domain_models.data_types.container_length import ContainerLength @@ -46,7 +47,8 @@ def setUp(self) -> None: LargeScheduledVehicle, Destination, Schedule, - TruckArrivalInformationForPickup + TruckArrivalInformationForPickup, + RandomSeedStore, ]) truck_arrival_distribution_seeder.seed() container_dwell_time_distribution_seeder.seed() diff --git a/conflowgen/tools/distribution_approximator.py b/conflowgen/tools/distribution_approximator.py index 41f81945..23b99376 100644 --- a/conflowgen/tools/distribution_approximator.py +++ b/conflowgen/tools/distribution_approximator.py @@ -6,6 +6,8 @@ import numpy as np +from conflowgen.application.repositories.random_seed_store_repository import get_initialised_random_object + class SamplerExhaustedException(Exception): """No more samples can be sampled from the sampler""" @@ -14,14 +16,13 @@ class SamplerExhaustedException(Exception): class DistributionApproximator: - random_seed = 1 + class_level_seeded_random: None | random.Random = None - def __init__(self, number_instances_per_category: Dict[any, int]) -> None: - """ - Args: - number_instances_per_category: For each key (category) the number of instances to draw is given - """ - self.seeded_random = random.Random(x=self.random_seed) + def __init__(self, number_instances_per_category: Dict[any, int], context_of_usage: str = "") -> None: + self.seeded_random = get_initialised_random_object( + self.__class__.__name__ + "__" + context_of_usage, + log_loading_process=False + ) self.target_distribution = np.array( list(number_instances_per_category.values()), dtype=np.int64 @@ -34,8 +35,16 @@ def __init__(self, number_instances_per_category: Dict[any, int]) -> None: def from_distribution( cls, distribution: Dict[any, float], - number_items: int + number_items: int, + context_of_usage: str = "" ) -> DistributionApproximator: + + if cls.class_level_seeded_random is None: + cls.class_level_seeded_random = get_initialised_random_object( + "DistributionApproximator__class", + log_loading_process=False + ) + assert math.isclose(sum(distribution.values()), 1, abs_tol=.001), \ f"All probabilities must sum to 1, but you only achieved {sum(distribution.values())}" @@ -49,9 +58,8 @@ def from_distribution( # Thus, we need to fill the missing items by randomly drawing some of them. number_items_in_category_estimation = sum(probability_based_instance_estimation.values()) if number_items_in_category_estimation < number_items: - seeded_random = random.Random(x=cls.random_seed) items_lost_to_rounding = number_items - number_items_in_category_estimation - randomly_chosen_categories = seeded_random.choices( + randomly_chosen_categories = cls.class_level_seeded_random.choices( population=list(distribution.keys()), weights=list(distribution.values()), k=items_lost_to_rounding @@ -59,7 +67,8 @@ def from_distribution( for category in randomly_chosen_categories: probability_based_instance_estimation[category] += 1 distribution_approximator = DistributionApproximator( - probability_based_instance_estimation + probability_based_instance_estimation, + context_of_usage=context_of_usage ) return distribution_approximator diff --git a/docs/conf.py b/docs/conf.py index acf44758..6590cf2f 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -71,7 +71,7 @@ 'Thumbs.db', '.DS_Store', # OS-specific '_build', # Sphinx-specific '.tools', # project-specific - '.ipynb_checkpoints', '.virtual_documents' # specific for Jupyter Notebooks + '**.ipynb_checkpoints', '**.virtual_documents' # specific for Jupyter Notebooks ] add_module_names = False diff --git a/examples/Python_Script/demo_DEHAM_CTA.py b/examples/Python_Script/demo_DEHAM_CTA.py index 18b55480..65b15e3a 100644 --- a/examples/Python_Script/demo_DEHAM_CTA.py +++ b/examples/Python_Script/demo_DEHAM_CTA.py @@ -87,7 +87,9 @@ logger.info(__doc__) # Pick database -database_chooser = conflowgen.DatabaseChooser() +database_chooser = conflowgen.DatabaseChooser( + sqlite_databases_directory=os.path.join(this_dir, "databases") +) demo_file_name = "demo_deham_cta.sqlite" database_chooser.create_new_sqlite_database( demo_file_name, diff --git a/examples/Python_Script/demo_poc.py b/examples/Python_Script/demo_poc.py index d14072a7..fb0a1844 100644 --- a/examples/Python_Script/demo_poc.py +++ b/examples/Python_Script/demo_poc.py @@ -37,7 +37,9 @@ logger.info(__doc__) # Pick database -database_chooser = conflowgen.DatabaseChooser() +database_chooser = conflowgen.DatabaseChooser( + sqlite_databases_directory=os.path.join(this_dir, "databases") +) demo_file_name = "demo_poc.sqlite" database_chooser.create_new_sqlite_database( demo_file_name,