Add files using upload-large-folder tool

basic_math_mj/part-00-00000.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:eb12a1a24233ecca23964732627d0d7719dcca9079a45beee39e298992941ab9
+size 396000

basic_math_mj/part-01-00000.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:398da4f497a6ea407f4a12e7b165d4bdabf8a64f62ab8f85c07a7b81888f64ec
+size 760000

basic_math_mj/part-02-00000.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e48a73961a7917c7540d5c228fe2af9eab6ff8ee6ed606e08ce342c9b7e42018
+size 910800

basic_math_mj/part-03-00000.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:898805438afed6d5f9e38a52bbdd0d1afdca49cff4d79c4f3f3479bc8edeab88
+size 3524400

basic_math_mj/part-04-00000.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d842d5c7bb00d2f09c2668f7c8b773bc38442b8ab76745b523f2ac9198ca59d8
+size 3400000

basic_math_mj/part-05-00000.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:63b61262ec144af65bc193e955c6f116f73e3746681e799c58783c6efbce1f57
+size 48048

basic_math_mj/part-06-00000.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9373a5436b942086bd1651d7797626d58935d40a50001f08d513266dca47bcf2
+size 77792

basic_math_mj/part-07-00000.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8ed70ee92a43f8daf9d5fc6cafd4e0aca5f7ab7b5534b8b6b25ce99b00a5c96e
+size 435600

basic_math_mj/part-08-00000.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d5d506f9bbfaf9ac3f76b07e6266c37882ec6f6066cb7d5d571dcbeb0d45ae3c
+size 320000

basic_math_mj/part-12-00000.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:613f2a71e840f6224daaa35511c8c9a612f3986b6ee6ee71ff693702904a36cf
+size 396000

basic_math_mj/part-13-00000.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:78bb6a0589faaf9b6591e5ee988041187129e3aa866e032ff37515a53ac1658e
+size 18304

basic_math_mj/part-17-00000.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8e01c772c71a448b275469ce969ac2a66e176505b346768705f2df0ceaea76fa
+size 752400

basic_math_mj/part-2-00000.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9aab445c68288b2d5d39e1feb7085ff60d6f7a299a9eab8577522e5b20a2be18
+size 534936

basic_math_mj/part-20-00000.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:03cb432710648e76158dc9d3409ef9df8575472d061164dd6dbe67cb1177a388
+size 1360000

basic_math_mj/part-21-00000.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:12447aa209babe103718605600f33404a39b1982f7edc8927743c9e1ff8e4c1e
+size 435600

basic_math_mj/part-22-00000.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3bf79ee0ce91534345a3d8573dd02c1f55f549361af4c9165252cbce5a45def0
+size 760000

basic_math_mj/part-23-00000.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1fccd92ec1544c538f06197e344b6f86e660dac08ed55d6bfb1262e763a288f1
+size 746440

basic_math_mj/part-24-00000.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:604392424d9265fa1bc20816390bbc9559dd7fa16a12a28d547a1bbf57b488ae
+size 346440

basic_math_mj/part-25-00000.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7d848e114a62c85fcf699a2186ff7ffe6a42b384c11f5d77ea2d8a567b39b1ed
+size 1386000

basic_math_mj/part-26-00000.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a30f031d4eb0dcaaca245165dae595321fe9b4adb9802d367db606f80eb5ff61
+size 194480

basic_math_mj/part-27-00000.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:50691ca6b081352fef3b286f20000a83f2012a0dae19c6919bd6ec4732e12b9a
+size 752400

basic_math_mj/part-28-00000.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1563d50e8fb4f0c6b5d8e407b2379de541bfa142736036087f1fcac68a0c916a
+size 720000

basic_math_mj/part-29-00000.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4b56b654c1256470e61d33ce38024f860c3a0abe97d5669b8dc39dc45f6531ae
+size 320000

basic_math_mj/part-3-00000.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dbce85344bc01085ce34bf87828d71a914cdd8ee4e527c1ac66b5062a4e98095
+size 605308

basic_math_mj/part-30-00000.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:42fa39241db16699c98e800b66d76c0837b9fe4012889b204f216fe288e41175
+size 1386440

basic_math_mj/part-31-00000.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f7da2180717d405b74401d3d23558b8f372b37d107739332dec004725f6178cf
+size 3505780

basic_math_mj/part-32-00000.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ddeb3f186ebf93635ab570d9ba9b7da1593e18793b6c9450144245e87cf79d02
+size 720000

basic_math_mj/part-33-00000.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:db07defda375f7a314aff04b89d974b3c85782205bb3f78713ddb849d7f5ce8f
+size 3400000

basic_math_mj/part-34-00000.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b54d25e1c39c1fbe646aff841759db289f87dbe4b93097e95a58ab12459a5d41
+size 346440

basic_math_mj/part-4-00000.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9701ca6e0e50ec634ee07e86af842920f1f30f73a83e933d36f8a5075d7beae3
+size 1640092

basic_math_mj/part-5-00000.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4ffeb5ada90594021464fcdce2716a0ce42376613fcdc7deca95527108d9172e
+size 3589284

dclm/part-2-00000.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:290fe9d3bf15a98ce8030e5f3f07dd07b863aa4985fd601a51a38aacece8b2ae
+size 35583812

download.sh

@@ -0,0 +1,65 @@
+#!/bin/bash
+
+# 输入文件路径（包含 URL 的文件）
+input_file="urls.txt"
+
+# 最大并发数（可以根据需要调整）
+max_concurrent=15
+
+# 检查输入文件是否存在
+if [[ ! -f "$input_file" ]]; then
+  echo "错误：文件 $input_file 不存在！"
+  exit 1
+fi
+
+# 计数器，用于跟踪当前正在运行的任务数量
+running_tasks=0
+
+# 逐行读取文件中的 URL
+while IFS= read -r url; do
+  # 检查 URL 是否为空
+  if [[ -z "$url" ]]; then
+    continue
+  fi
+
+  # 提取 preprocessed 后面的字段作为目录名
+  directory=$(echo "$url" | grep -oP '(?<=preprocessed/)[^/]+')
+
+  # 如果提取不到目录名，跳过该 URL
+  if [[ -z "$directory" ]]; then
+    echo "警告：URL $url 缺少有效的目录信息，跳过！"
+    continue
+  fi
+
+  # 创建目标目录（如果不存在）
+  mkdir -p "$directory"
+
+  # 提取文件名
+  filename=$(basename "$url")
+
+  # 下载文件到目标目录（并发执行）
+  (
+    echo "正在下载 $url 到 $directory/$filename ..."
+    wget -q -O "$directory/$filename" "$url"
+    if [[ $? -eq 0 ]]; then
+      echo "下载成功：$directory/$filename"
+    else
+      echo "下载失败：$url"
+    fi
+  ) &
+
+  # 更新并发任务计数器
+  running_tasks=$((running_tasks + 1))
+
+  # 如果达到最大并发数，等待部分任务完成
+  if [[ $running_tasks -ge $max_concurrent ]]; then
+    wait -n  # 等待任意一个后台任务完成
+    running_tasks=$((running_tasks - 1))
+  fi
+
+done < "$input_file"
+
+# 等待所有剩余的任务完成
+wait
+
+echo "所有文件处理完成！"

mathcoder2-synthmath/part-00-00000.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:78d5f9ca2382d57ea568366b0239c6dc6974561241f37aef8391fb0fca1e88ab
+size 128877412

mathcoder2-synthmath/part-01-00000.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0d03a08107ac028f3390da5b18fb5b3dffe76233c870e9a80c81cbe6652bfdd6
+size 108442852

mathcoder2-synthmath/part-02-00000.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9ed2b4374e76e08ad1d1b6315d7b4ec078c324ad9b690e470f745772dbe53a43
+size 116948080

mathcoder2-synthmath/part-03-00000.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0c399da3f1db9f38b1333e6a7f671ea2c7f21477de797deae594e47cadf1ec12
+size 100188860

mathcoder2-synthmath/part-04-00000.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e5427e06cf74581aada0e497a9d44d5b77f510c1803dfd3b4814d150f0d8efc5
+size 151711548

mathcoder2-synthmath/part-05-00000.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f3aaa01f05b985b665003de8a7ac5efeb56dd2ea9d7eb2f26040f7db84a0963c
+size 517088260

mathcoder2-synthmath/part-06-00000.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bc31b1cbd0d9ffd01393612208b6ba6fe28dc7b25cdf1e993ca12a4991128c3d
+size 139817528

mathcoder2-synthmath/part-07-00000.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:408d7ef57f954067e84c339a83e5e0786bc77555f80f3f9bf452d8eff0beea82
+size 108618344

mathcoder2-synthmath/part-08-00000.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f62530bc95bb46590b04bc6b03a7303cf1ec6353fa84bb1d4f9f1f34f86b0bc7
+size 518112528

mathcoder2-synthmath/part-09-00000.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:92ebbafa606fdc8f3da787dd9e8c306856a2901a4efbff9f53fa5498c0b884e7
+size 133336460

mathcoder2-synthmath/part-10-00000.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2006f18a0a719755ea25465c8c0d1824b2569a89447296d6a581b7749227057e
+size 109192520

mathcoder2-synthmath/part-11-00000.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0b6bc563fe8f40b195f14cab0b3af0a4e4cdb577cfac4e1c592d8a2b25a0967d
+size 151643232

mathcoder2-synthmath/part-12-00000.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:466f4ce4a15cd7e4266ff2475f9977c2edff8f1f13b74b2e5f84ff580ecf080a
+size 127514692

mathcoder2-synthmath/part-13-00000.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3c05e1c15f898e89b917efec3d4a24c03cb2062f0202afaa7c7622a63aefe79b
+size 152834308

mathcoder2-synthmath/part-14-00000.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3168772336a2a63298b4fa333de1f3bb329d12e04889be42d5a8c1564a24335a
+size 197948716

mathcoder2-synthmath/part-15-00000.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c985a20fd187b5f4bd70bafc77a4314f0ecca1616d39c177d2f0cba6156c54bd
+size 163433248

mathcoder2-synthmath/part-16-00000.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:246830d89901d0347e912515fdafd8dc88118c45bddf2a8ee25736734a4d7552
+size 130902512