Added git-filter-repo

pyproject.toml

@@ -18,6 +18,7 @@ dependencies = [
     "fast-langdetect>=1.0.0",
     "fastapi>=0.117.1",
     "generateds>=2.44.3",
+    "git-filter-repo>=2.47.0",
     "httpx>=0.28.1",
     "lxml>=6.0.1",
     "pandas>=2.3.3",

uv.lock

@@ -42,6 +42,7 @@ dependencies = [
     { name = "fast-langdetect" },
     { name = "fastapi" },
     { name = "generateds" },
+    { name = "git-filter-repo" },
     { name = "httpx" },
     { name = "lxml" },
     { name = "pandas" },
@@ -75,6 +76,7 @@ requires-dist = [
     { name = "fast-langdetect", specifier = ">=1.0.0" },
     { name = "fastapi", specifier = ">=0.117.1" },
     { name = "generateds", specifier = ">=2.44.3" },
+    { name = "git-filter-repo", specifier = ">=2.47.0" },
     { name = "httpx", specifier = ">=0.28.1" },
     { name = "lxml", specifier = ">=6.0.1" },
     { name = "pandas", specifier = ">=2.3.3" },
@@ -516,6 +518,15 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/b2/84/79ca1e01337fe898cd303ac8d51151b4bea4891028b93ae5bf5e9cc911a9/generateDS-2.44.3-py3-none-any.whl", hash = "sha256:ae5db7105ca777182ba6549118c9aba1690ea341400af13ffbdbfbe1bc022299", size = 147394, upload-time = "2024-10-08T21:54:34.506Z" },
 ]
 
+[[package]]
+name = "git-filter-repo"
+version = "2.47.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/b9/bb/7a283f568af6b0528ade65e8ace84bd6ba46003e429101bcd62c232d01a5/git_filter_repo-2.47.0.tar.gz", hash = "sha256:411b27e68a080c07a69c233cb526dbc2d848b09a72f10477f4444dd0822cf290", size = 275743, upload-time = "2024-12-04T03:10:48.2Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/60/60/d3943f0880ebcb7e0bdf79254d10dddd39c7b656eeecae32b8806ff66dec/git_filter_repo-2.47.0-py3-none-any.whl", hash = "sha256:2cd04929b9024e83e65db571cbe36aec65ead0cb5f9ec5abe42158654af5ad83", size = 76282, upload-time = "2024-12-04T03:10:46.064Z" },
+]
+
 [[package]]
 name = "greenlet"
 version = "3.2.4"