fix: Ensure consistent data storage by explicitly using \n for newlines when storing files (#1601)
<!-- .github/pull_request_template.md --> ## Description <!-- Please provide a clear, human-generated description of the changes in this PR. DO NOT use AI-generated descriptions. We want to understand your thought process and reasoning. --> `LocalFileStorage.store(...)` when storing strings, uses system defaults for newlines. It is universally `\n` in Unix and MacOS, but for Windows by default its `\r\n` (ancient Windows standard) This results in hash mismatch, although content is identical. Windows can work with `\n`, so this PR explicitly uses `\n` for newline encoding ## Type of Change <!-- Please check the relevant option --> - [ ] Bug fix (non-breaking change that fixes an issue) - [ ] New feature (non-breaking change that adds functionality) - [ ] Breaking change (fix or feature that would cause existing functionality to change) - [ ] Documentation update - [ ] Code refactoring - [ ] Performance improvement - [ ] Other (please specify): ## Screenshots/Videos (if applicable) <!-- Add screenshots or videos to help explain your changes --> ## Pre-submission Checklist <!-- Please check all boxes that apply before submitting your PR --> - [ ] **I have tested my changes thoroughly before submitting this PR** - [ ] **This PR contains minimal changes necessary to address the issue/feature** - [ ] My code follows the project's coding standards and style guidelines - [ ] I have added tests that prove my fix is effective or that my feature works - [ ] I have added necessary documentation (if applicable) - [ ] All new and existing tests pass - [ ] I have searched existing PRs to ensure this change hasn't been submitted already - [ ] I have linked any relevant issues in the description - [ ] My commits have clear and descriptive messages ## DCO Affirmation I affirm that all code in every commit of this pull request conforms to the terms of the Topoteretes Developer Certificate of Origin.
This commit is contained in:
commit
4db41ca1da
2 changed files with 20 additions and 20 deletions
|
|
@ -82,16 +82,16 @@ class LocalFileStorage(Storage):
|
||||||
self.ensure_directory_exists(file_dir_path)
|
self.ensure_directory_exists(file_dir_path)
|
||||||
|
|
||||||
if overwrite or not os.path.exists(full_file_path):
|
if overwrite or not os.path.exists(full_file_path):
|
||||||
with open(
|
if isinstance(data, str):
|
||||||
full_file_path,
|
with open(full_file_path, mode="w", encoding="utf-8", newline="\n") as file:
|
||||||
mode="w" if isinstance(data, str) else "wb",
|
|
||||||
encoding="utf-8" if isinstance(data, str) else None,
|
|
||||||
) as file:
|
|
||||||
if hasattr(data, "read"):
|
|
||||||
data.seek(0)
|
|
||||||
file.write(data.read())
|
|
||||||
else:
|
|
||||||
file.write(data)
|
file.write(data)
|
||||||
|
else:
|
||||||
|
with open(full_file_path, mode="wb") as file:
|
||||||
|
if hasattr(data, "read"):
|
||||||
|
data.seek(0)
|
||||||
|
file.write(data.read())
|
||||||
|
else:
|
||||||
|
file.write(data)
|
||||||
|
|
||||||
file.close()
|
file.close()
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -70,18 +70,18 @@ class S3FileStorage(Storage):
|
||||||
if overwrite or not await self.file_exists(file_path):
|
if overwrite or not await self.file_exists(file_path):
|
||||||
|
|
||||||
def save_data_to_file():
|
def save_data_to_file():
|
||||||
with self.s3.open(
|
if isinstance(data, str):
|
||||||
full_file_path,
|
with self.s3.open(
|
||||||
mode="w" if isinstance(data, str) else "wb",
|
full_file_path, mode="w", encoding="utf-8", newline="\n"
|
||||||
encoding="utf-8" if isinstance(data, str) else None,
|
) as file:
|
||||||
) as file:
|
|
||||||
if hasattr(data, "read"):
|
|
||||||
data.seek(0)
|
|
||||||
file.write(data.read())
|
|
||||||
else:
|
|
||||||
file.write(data)
|
file.write(data)
|
||||||
|
else:
|
||||||
file.close()
|
with self.s3.open(full_file_path, mode="wb") as file:
|
||||||
|
if hasattr(data, "read"):
|
||||||
|
data.seek(0)
|
||||||
|
file.write(data.read())
|
||||||
|
else:
|
||||||
|
file.write(data)
|
||||||
|
|
||||||
await run_async(save_data_to_file)
|
await run_async(save_data_to_file)
|
||||||
|
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue