From f91d49d987065df43cbaa8264d349885f001aa17 Mon Sep 17 00:00:00 2001 From: scragly <29337040+scragly@users.noreply.github.com> Date: Thu, 19 Sep 2019 22:59:46 +1000 Subject: Add persistent datafile utils. --- bot/utils/persist.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 bot/utils/persist.py (limited to 'bot/utils/persist.py') diff --git a/bot/utils/persist.py b/bot/utils/persist.py new file mode 100644 index 00000000..ec6f306a --- /dev/null +++ b/bot/utils/persist.py @@ -0,0 +1,24 @@ +import sqlite3 +from pathlib import Path +from shutil import copyfile + +DIRECTORY = Path("data") # directory that has a persistent volume mapped to it + + +def datafile(file_path: Path) -> Path: + """Copy datafile at the provided file_path to the persistent data directory.""" + if not file_path.exists(): + raise OSError(f"File not found at {file_path}.") + + persistant_path = Path(DIRECTORY, file_path.name) + + if not persistant_path.exists(): + copyfile(file_path, persistant_path) + + return persistant_path + + +def sqlite(db_path: Path) -> sqlite3.Connection: + """Copy sqlite file to the persistent data directory and return an open connection.""" + persistant_path = datafile(db_path) + return sqlite3.connect(persistant_path) -- cgit v1.2.3 From 4d7c93296d67f182b849d2a5227d692640452085 Mon Sep 17 00:00:00 2001 From: scragly <29337040+scragly@users.noreply.github.com> Date: Fri, 20 Sep 2019 11:10:30 +1000 Subject: Add better explanatory docstring and example for persist.datafile. --- bot/utils/persist.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) (limited to 'bot/utils/persist.py') diff --git a/bot/utils/persist.py b/bot/utils/persist.py index ec6f306a..06c3764a 100644 --- a/bot/utils/persist.py +++ b/bot/utils/persist.py @@ -6,7 +6,23 @@ DIRECTORY = Path("data") # directory that has a persistent volume mapped to it def datafile(file_path: Path) -> Path: - """Copy datafile at the provided file_path to the persistent data directory.""" + """ + Copy datafile at the provided file_path to the persistent data directory. + + A persistent data file is needed by some features in order to not lose data + after bot rebuilds. + + This function will ensure that a clean data file with default schema, + structure or data is copied over to the persistent volume before returning + the path to this new persistent version of the file. + + If the persistent file already exists, it won't be overwritten with the + clean default file, just returning the Path instead to the existing file. + + Example Usage: + >>> clean_default_datafile = Path("bot", "resources", "datafile.json") + >>> persistent_file_path = datafile(clean_default_datafile) + """ if not file_path.exists(): raise OSError(f"File not found at {file_path}.") -- cgit v1.2.3 From b385db0f08fb8bb3d46cf8f820d5dd525d7b2272 Mon Sep 17 00:00:00 2001 From: scragly <29337040+scragly@users.noreply.github.com> Date: Mon, 23 Sep 2019 12:01:57 +1000 Subject: Check explicitly if file exists rather than any existing path. Co-Authored-By: Mark --- bot/utils/persist.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'bot/utils/persist.py') diff --git a/bot/utils/persist.py b/bot/utils/persist.py index 06c3764a..35e1e41a 100644 --- a/bot/utils/persist.py +++ b/bot/utils/persist.py @@ -23,7 +23,7 @@ def datafile(file_path: Path) -> Path: >>> clean_default_datafile = Path("bot", "resources", "datafile.json") >>> persistent_file_path = datafile(clean_default_datafile) """ - if not file_path.exists(): + if not file_path.is_file(): raise OSError(f"File not found at {file_path}.") persistant_path = Path(DIRECTORY, file_path.name) -- cgit v1.2.3 From c1b8b23e4a327631f287b1dc68ab76967651e57e Mon Sep 17 00:00:00 2001 From: scragly <29337040+scragly@users.noreply.github.com> Date: Mon, 23 Sep 2019 16:12:18 +1000 Subject: Improve func name, example, directory management Function name has been changed to `make_persistent` after prompt by @lemonsaurus asking for a more descriptive name. Thanks @MarkKoz for providing the alternate name. During local testing, the `data` directory doesn't exist yet. In prod, this isn't an issue as the persistent volume is mounted at that location. To make local testing more convenient, the directory is checked and made if not found. Persistent data files will be placed in a seasonal subdirectory so long as they have a valid season name somewhere in their path, otherwise they will be placed directly in the data directory. Added a note to docstring to avoid same-named files in the same seasons or it will conflict with each other in the persistent data directory. The example was extended a little bit to make it both actually valid if tested and hopefully make it easier to understand what's going on. --- bot/utils/persist.py | 46 +++++++++++++++++++++++++++++++++++++--------- 1 file changed, 37 insertions(+), 9 deletions(-) (limited to 'bot/utils/persist.py') diff --git a/bot/utils/persist.py b/bot/utils/persist.py index 35e1e41a..939a95c9 100644 --- a/bot/utils/persist.py +++ b/bot/utils/persist.py @@ -2,10 +2,12 @@ import sqlite3 from pathlib import Path from shutil import copyfile +from bot.seasons.season import get_seasons + DIRECTORY = Path("data") # directory that has a persistent volume mapped to it -def datafile(file_path: Path) -> Path: +def make_persistent(file_path: Path) -> Path: """ Copy datafile at the provided file_path to the persistent data directory. @@ -19,22 +21,48 @@ def datafile(file_path: Path) -> Path: If the persistent file already exists, it won't be overwritten with the clean default file, just returning the Path instead to the existing file. + Note: Avoid using the same file name as other features in the same seasons + as otherwise only one datafile can be persistent and will be returned for + both cases. + Example Usage: - >>> clean_default_datafile = Path("bot", "resources", "datafile.json") - >>> persistent_file_path = datafile(clean_default_datafile) + >>> import json + >>> template_datafile = Path("bot", "resources", "evergreen", "myfile.json") + >>> path_to_persistent_file = make_persistent(template_datafile) + >>> print(path_to_persistent_file) + data/evergreen/myfile.json + >>> with path_to_persistent_file.open("w+") as f: + >>> data = json.load(f) """ + # ensure the persistent data directory exists + if not DIRECTORY.exists(): + DIRECTORY.mkdir() + if not file_path.is_file(): raise OSError(f"File not found at {file_path}.") - persistant_path = Path(DIRECTORY, file_path.name) + # detect season in datafile path for assigning to subdirectory + season = next((s for s in get_seasons() if s in file_path.parts), None) + + if season: + # make sure subdirectory exists first + subdirectory = Path(DIRECTORY, season) + if not subdirectory.exists(): + subdirectory.mkdir() + + persistent_path = Path(subdirectory, file_path.name) + + else: + persistent_path = Path(DIRECTORY, file_path.name) - if not persistant_path.exists(): - copyfile(file_path, persistant_path) + # copy base/template datafile to persistent directory + if not persistent_path.exists(): + copyfile(file_path, persistent_path) - return persistant_path + return persistent_path def sqlite(db_path: Path) -> sqlite3.Connection: """Copy sqlite file to the persistent data directory and return an open connection.""" - persistant_path = datafile(db_path) - return sqlite3.connect(persistant_path) + persistent_path = make_persistent(db_path) + return sqlite3.connect(persistent_path) -- cgit v1.2.3 From 3ea2130d71b69bd733675c43f91d7d081472735c Mon Sep 17 00:00:00 2001 From: scragly <29337040+scragly@users.noreply.github.com> Date: Mon, 23 Sep 2019 17:34:30 +1000 Subject: Use mkdir exists kwarg instead of checking existing ahead of time. --- bot/utils/persist.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) (limited to 'bot/utils/persist.py') diff --git a/bot/utils/persist.py b/bot/utils/persist.py index 939a95c9..a60a1219 100644 --- a/bot/utils/persist.py +++ b/bot/utils/persist.py @@ -35,8 +35,7 @@ def make_persistent(file_path: Path) -> Path: >>> data = json.load(f) """ # ensure the persistent data directory exists - if not DIRECTORY.exists(): - DIRECTORY.mkdir() + DIRECTORY.mkdir(exist_ok=True) if not file_path.is_file(): raise OSError(f"File not found at {file_path}.") @@ -47,8 +46,7 @@ def make_persistent(file_path: Path) -> Path: if season: # make sure subdirectory exists first subdirectory = Path(DIRECTORY, season) - if not subdirectory.exists(): - subdirectory.mkdir() + subdirectory.mkdir(exist_ok=True) persistent_path = Path(subdirectory, file_path.name) -- cgit v1.2.3