CI: Generate SF1 during CI from scratch
@@ -45,23 +45,39 @@ jobs:
wget -q https://ldbcouncil.org/ldbc_snb_datagen_spark/social-network-sf0.003-bi-composite-merged-fk.zip
unzip -q social-network-sf0.003-bi-composite-merged-fk.zip
- run:
- name: Convert update streams
+ name: Convert SF0.003 update streams
command: |
git clone https://github.com/ldbc/ldbc_snb_interactive_driver
cd ldbc_snb_interactive_driver/scripts/
- ./install-dependencies-for-convert-script.sh
+ ./install-dependencies.sh
export LDBC_SNB_DATA_ROOT_DIRECTORY=../../social-network-sf0.003-bi-composite-merged-fk/
./convert.sh
mv inserts ../../update-streams/
mv deletes ../../update-streams/
cd ../..
- run:
- name: Generate parameters
+ name: Generate parameters for SF0.003
command: |
cd paramgen
scripts/get-sample-factors.sh
scripts/paramgen.sh
cd ..
+ ### Generate SF1 data sets
+ - run:
+ name: Clone Datagen repository
+ command: |
+ git clone git@github.com:ldbc/ldbc_snb_datagen_spark.git
+ curl https://archive.apache.org/dist/spark/spark-3.2.2/spark-3.2.2-bin-hadoop3.2.tgz | tar -xz -C ~/
+ - run:
+ name: Generate SF1 data sets
+ command: |
+ export LDBC_SNB_DATAGEN_MAX_MEM=8G
+ export LDBC_SNB_DATAGEN_DIR=`pwd`/ldbc_snb_datagen_spark
+ export LDBC_SNB_DRIVER_DIR=`pwd`/ldbc_snb_interactive_driver
+ export SPARK_HOME=~/spark-3.2.2-bin-hadoop3.2
+ export PATH="${SPARK_HOME}/bin":"${PATH}"
+ export SF=1
+ scripts/generate-all.sh
### Neo4j
- run:
name: Tiny data set - Load Neo4j database and create a backup
@@ -130,38 +146,6 @@ jobs:
driver/benchmark.sh
scripts/stop.sh
cd ..
- - run:
- name: Get SF1 data set
- command: |
- curl -L https://r2-public-worker.ldbc.workers.dev/interactive/snb-sf1-2022-07-11.tar.zst | tar -x --use-compress-program=unzstd
- - run:
- name: Process SF1 data set
- command: |
- export SF=1
- export LDBC_SNB_DRIVER_DIR=`pwd`/ldbc_snb_interactive_driver
- export LDBC_SNB_DATA_ROOT_DIRECTORY=`pwd`/out-sf${SF}
- export LDBC_SNB_IMPLS_DIR=`pwd`
-
- echo "==================== Generate update streams ===================="
- cd ${LDBC_SNB_IMPLS_DIR}
- mkdir update-streams-sf${SF}/
-
- cd ${LDBC_SNB_DRIVER_DIR}
- cd scripts
- ./convert.sh
- mv inserts ${LDBC_SNB_IMPLS_DIR}/update-streams-sf${SF}/
- mv deletes ${LDBC_SNB_IMPLS_DIR}/update-streams-sf${SF}/
-
- echo "==================== Generate parameters ===================="
- cd ${LDBC_SNB_IMPLS_DIR}
- mkdir parameters-sf${SF}
-
- cd paramgen
- scripts/get-factors.sh
- scripts/paramgen.sh
- cd ..
-
- mv parameters/*.parquet parameters-sf${SF}/
### Neo4j
- run:
name: SF1 data set - Load Neo4j database and create a backup
