yonikremer commited on
Commit
d3e85c8
1 Parent(s): 0a466f9

downloading only the pytorch model and important files, not the other versions of the model

Browse files
Files changed (1) hide show
  1. download_repo.py +18 -3
download_repo.py CHANGED
@@ -15,8 +15,11 @@ def change_default_timeout(new_timeout: int) -> None:
15
  urllib3.util.timeout.DEFAULT_TIMEOUT = new_timeout
16
 
17
 
18
- def download_repository(name: str) -> None:
19
- """Downloads a repository from the Hugging Face Hub."""
 
 
 
20
  number_of_seconds_in_a_day: int = 86_400
21
  change_default_timeout(number_of_seconds_in_a_day)
22
  curr_folder: str = os.path.dirname(__file__)
@@ -26,5 +29,17 @@ def download_repository(name: str) -> None:
26
  etag_timeout=number_of_seconds_in_a_day,
27
  resume_download=True,
28
  repo_type="model",
29
- library_name="pt"
 
 
 
 
 
 
 
 
30
  )
 
 
 
 
 
15
  urllib3.util.timeout.DEFAULT_TIMEOUT = new_timeout
16
 
17
 
18
+ def download_pytorch_model(name: str) -> None:
19
+ """
20
+ Downloads a pytorch model and all the small files from the model's repository.
21
+ Other model formats (tensorflow, tflite, safetensors, msgpack and ot) are not downloaded.
22
+ """
23
  number_of_seconds_in_a_day: int = 86_400
24
  change_default_timeout(number_of_seconds_in_a_day)
25
  curr_folder: str = os.path.dirname(__file__)
 
29
  etag_timeout=number_of_seconds_in_a_day,
30
  resume_download=True,
31
  repo_type="model",
32
+ library_name="pt",
33
+ # h5, tflite, safetensors, msgpack and ot models files are not needed
34
+ ignore_patterns=[
35
+ "*.h5",
36
+ "*.tflite",
37
+ "*.safetensors",
38
+ "*.msgpack",
39
+ "*.ot",
40
+ ],
41
  )
42
+
43
+
44
+ if __name__ == "__main__":
45
+ download_pytorch_model("facebook/opt-125m")