Merge pull request ipfs#9657 from laurentsenta/test/sharness-with-car…

…-fixtures

test(gateway): use deterministic CAR fixtures

test/sharness/t0113-gateway-symlink.sh

@@ -9,26 +9,21 @@ test_description="Test symlink support on the HTTP gateway"
 test_init_ipfs
 test_launch_ipfs_daemon
 
-
-test_expect_success "Create a test directory with symlinks" '
-  mkdir testfiles &&
-  echo "content" > testfiles/foo &&
-  ln -s foo testfiles/bar &&
-  test_cmp testfiles/foo testfiles/bar
-'
-
-test_expect_success "Add the test directory" '
-  HASH=$(ipfs add -Qr testfiles)
+# Import test case
+# See the static fixtures in ./t0113-gateway-symlink/
+test_expect_success "Add the test directory with symlinks" '
+  ipfs dag import ../t0113-gateway-symlink/testfiles.car
 '
+ROOT_DIR_CID=QmWvY6FaqFMS89YAQ9NAPjVP4WZKA1qbHbicc9HeSKQTgt # ./testfiles/
 
 test_expect_success "Test the directory listing" '
-  curl "$GWAY_ADDR/ipfs/$HASH/" > list_response &&
+  curl "$GWAY_ADDR/ipfs/$ROOT_DIR_CID/" > list_response &&
   test_should_contain ">foo<" list_response &&
   test_should_contain ">bar<" list_response
 '
 
 test_expect_success "Test the symlink" '
-  curl "$GWAY_ADDR/ipfs/$HASH/bar" > bar_actual &&
+  curl "$GWAY_ADDR/ipfs/$ROOT_DIR_CID/bar" > bar_actual &&
   echo -n "foo" > bar_expected &&
   test_cmp bar_expected bar_actual
 '

test/sharness/t0113-gateway-symlink/README.md

@@ -0,0 +1,17 @@
+# Dataset description/sources
+
+- testfiles.car
+  - raw CARv1
+
+generated with:
+
+```sh
+# using ipfs version 0.18.1
+mkdir testfiles &&
+echo "content" > testfiles/foo &&
+ln -s foo testfiles/bar &&
+ROOT_DIR_CID=$(ipfs add -Qr testfiles) &&
+ipfs dag export $ROOT_DIR_CID > testfiles.car
+
+# ROOT_DIR_CID=QmWvY6FaqFMS89YAQ9NAPjVP4WZKA1qbHbicc9HeSKQTgt
+```

test/sharness/t0114-gateway-subdomains.sh

@@ -91,27 +91,19 @@ test_expect_success "ipfs init" '
 
 test_launch_ipfs_daemon_without_network
 
+# Import test case
+# See the static fixtures in ./t0114-gateway-subdomains/
+test_expect_success "Add the test fixtures" '
+  ipfs dag import ../t0114-gateway-subdomains/fixtures.car
+'
+CID_VAL="hello"
+CIDv1=bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am
+CIDv0=QmZULkCELmmk5XNfCgTnCyFgAVxBRBXyDHGGMVoLFLiXEN
 # CIDv0to1 is necessary because raw-leaves are enabled by default during
 # "ipfs add" with CIDv1 and disabled with CIDv0
-test_expect_success "Add test text file" '
-  CID_VAL="hello"
-  CIDv1=$(echo $CID_VAL | ipfs add --cid-version 1 -Q)
-  CIDv0=$(echo $CID_VAL | ipfs add --cid-version 0 -Q)
-  CIDv0to1=$(echo "$CIDv0" | ipfs cid base32)
-  echo CIDv0to1=${CIDv0to1}
-'
-
-# Directory tree crafted to test for edge cases like "/ipfs/ipfs/ipns/bar"
-test_expect_success "Add the test directory" '
-  mkdir -p testdirlisting/ipfs/ipns &&
-  echo "hello" > testdirlisting/hello &&
-  echo "text-file-content" > testdirlisting/ipfs/ipns/bar &&
-  mkdir -p testdirlisting/api &&
-  mkdir -p testdirlisting/ipfs &&
-  echo "I am a txt file" > testdirlisting/api/file.txt &&
-  echo "I am a txt file" > testdirlisting/ipfs/file.txt &&
-  DIR_CID=$(ipfs add -Qr --cid-version 1 testdirlisting)
-'
+CIDv0to1=bafybeiffndsajwhk3lwjewwdxqntmjm4b5wxaaanokonsggenkbw6slwk4
+CIDv1_TOO_LONG=bafkrgqhhyivzstcz3hhswshfjgy6ertgmnqeleynhwt4dlfsthi4hn7zgh4uvlsb5xncykzapi3ocd4lzogukir6ksdy6wzrnz6ohnv4aglcs
+DIR_CID=bafybeiht6dtwk3les7vqm6ibpvz6qpohidvlshsfyr7l5mpysdw2vmbbhe # ./testdirlisting
 
 test_expect_success "Publish test text file to IPNS using RSA keys" '
   RSA_KEY=$(ipfs key gen --ipns-base=b58mh --type=rsa --size=2048 test_key_rsa | head -n1 | tr -d "\n")
@@ -600,8 +592,6 @@ test_expect_success \
 IPNS_KEY="test_key_ed25519"
 IPNS_ED25519_B58MH=$(ipfs key list -l --ipns-base b58mh | grep $IPNS_KEY | cut -d" " -f1 | tr -d "\n")
 IPNS_ED25519_B36CID=$(ipfs key list -l --ipns-base base36 | grep $IPNS_KEY | cut -d" " -f1 | tr -d "\n")
-# sha512 will be over 63char limit, even when represented in Base36
-CIDv1_TOO_LONG=$(echo $CID_VAL | ipfs add --cid-version 1 --hash sha2-512 -Q)
 
 # local: *.localhost
 test_localhost_gateway_response_should_contain \

test/sharness/t0114-gateway-subdomains/README.md

@@ -0,0 +1,54 @@
+# Dataset description/sources
+
+- fixtures.car
+  - raw CARv1
+
+generated with:
+
+```sh
+# using ipfs version 0.18.1
+
+# CIDv0to1 is necessary because raw-leaves are enabled by default during
+# "ipfs add" with CIDv1 and disabled with CIDv0
+CID_VAL="hello"
+CIDv1=$(echo $CID_VAL | ipfs add --cid-version 1 -Q)
+CIDv0=$(echo $CID_VAL | ipfs add --cid-version 0 -Q)
+CIDv0to1=$(echo "$CIDv0" | ipfs cid base32)
+# sha512 will be over 63char limit, even when represented in Base36
+CIDv1_TOO_LONG=$(echo $CID_VAL | ipfs add --cid-version 1 --hash sha2-512 -Q)
+
+echo CIDv1=${CIDv1}
+echo CIDv0=${CIDv0}
+echo CIDv0to1=${CIDv0to1}
+echo CIDv1_TOO_LONG=${CIDv1_TOO_LONG}
+
+# Directory tree crafted to test for edge cases like "/ipfs/ipfs/ipns/bar"
+mkdir -p testdirlisting/ipfs/ipns &&
+echo "hello" > testdirlisting/hello &&
+echo "text-file-content" > testdirlisting/ipfs/ipns/bar &&
+mkdir -p testdirlisting/api &&
+mkdir -p testdirlisting/ipfs &&
+echo "I am a txt file" > testdirlisting/api/file.txt &&
+echo "I am a txt file" > testdirlisting/ipfs/file.txt &&
+DIR_CID=$(ipfs add -Qr --cid-version 1 testdirlisting)
+
+echo DIR_CID=${DIR_CID}
+
+ipfs files mkdir /t0114/
+ipfs files cp /ipfs/${CIDv1} /t0114/
+ipfs files cp /ipfs/${CIDv0} /t0114/
+ipfs files cp /ipfs/${CIDv0to1} /t0114/
+ipfs files cp /ipfs/${DIR_CID} /t0114/
+ipfs files cp /ipfs/${CIDv1_TOO_LONG} /t0114/
+
+ROOT=`ipfs files stat /t0114/ --hash`
+
+ipfs dag export ${ROOT} > ./fixtures.car
+
+# CID_VAL="hello"
+# CIDv1=bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am
+# CIDv0=QmZULkCELmmk5XNfCgTnCyFgAVxBRBXyDHGGMVoLFLiXEN
+# CIDv0to1=bafybeiffndsajwhk3lwjewwdxqntmjm4b5wxaaanokonsggenkbw6slwk4
+# CIDv1_TOO_LONG=bafkrgqhhyivzstcz3hhswshfjgy6ertgmnqeleynhwt4dlfsthi4hn7zgh4uvlsb5xncykzapi3ocd4lzogukir6ksdy6wzrnz6ohnv4aglcs
+# DIR_CID=bafybeiht6dtwk3les7vqm6ibpvz6qpohidvlshsfyr7l5mpysdw2vmbbhe # ./testdirlisting
+```

test/sharness/t0115-gateway-dir-listing.sh

@@ -18,20 +18,14 @@ test_expect_success "ipfs init" '
 
 test_launch_ipfs_daemon_without_network
 
+# Import test case
+# See the static fixtures in ./t0115-gateway-dir-listing/
 test_expect_success "Add the test directory" '
-  mkdir -p rootDir/ipfs &&
-  mkdir -p rootDir/ipns &&
-  mkdir -p rootDir/api &&
-  mkdir -p rootDir/ą/ę &&
-  echo "I am a txt file on path with utf8" > rootDir/ą/ę/file-źł.txt &&
-  echo "I am a txt file in confusing /api dir" > rootDir/api/file.txt &&
-  echo "I am a txt file in confusing /ipfs dir" > rootDir/ipfs/file.txt &&
-  echo "I am a txt file in confusing /ipns dir" > rootDir/ipns/file.txt &&
-  DIR_CID=$(ipfs add -Qr --cid-version 1 rootDir) &&
-  FILE_CID=$(ipfs files stat --enc=json /ipfs/$DIR_CID/ą/ę/file-źł.txt | jq -r .Hash) &&
-  FILE_SIZE=$(ipfs files stat --enc=json /ipfs/$DIR_CID/ą/ę/file-źł.txt | jq -r .Size)
-  echo "$FILE_CID / $FILE_SIZE"
+  ipfs dag import ../t0115-gateway-dir-listing/fixtures.car
 '
+DIR_CID=bafybeig6ka5mlwkl4subqhaiatalkcleo4jgnr3hqwvpmsqfca27cijp3i # ./rootDir/
+FILE_CID=bafkreialihlqnf5uwo4byh4n3cmwlntwqzxxs2fg5vanqdi3d7tb2l5xkm # ./rootDir/ą/ę/file-źł.txt
+FILE_SIZE=34
 
 ## ============================================================================
 ## Test dir listing on path gateway (eg. 127.0.0.1:8080/ipfs/)

test/sharness/t0115-gateway-dir-listing/README.md

@@ -0,0 +1,32 @@
+# Dataset description/sources
+
+- fixtures.car
+  - raw CARv1
+
+generated with:
+
+```sh
+# using ipfs version 0.18.1
+mkdir -p rootDir/ipfs &&
+mkdir -p rootDir/ipns &&
+mkdir -p rootDir/api &&
+mkdir -p rootDir/ą/ę &&
+echo "I am a txt file on path with utf8" > rootDir/ą/ę/file-źł.txt &&
+echo "I am a txt file in confusing /api dir" > rootDir/api/file.txt &&
+echo "I am a txt file in confusing /ipfs dir" > rootDir/ipfs/file.txt &&
+echo "I am a txt file in confusing /ipns dir" > rootDir/ipns/file.txt &&
+DIR_CID=$(ipfs add -Qr --cid-version 1 rootDir) &&
+FILE_CID=$(ipfs files stat --enc=json /ipfs/$DIR_CID/ą/ę/file-źł.txt | jq -r .Hash) &&
+FILE_SIZE=$(ipfs files stat --enc=json /ipfs/$DIR_CID/ą/ę/file-źł.txt | jq -r .Size)
+echo "$FILE_CID / $FILE_SIZE"
+
+echo DIR_CID=${DIR_CID}
+echo FILE_CID=${FILE_CID}
+echo FILE_SIZE=${FILE_SIZE}
+
+ipfs dag export ${DIR_CID} > ./fixtures.car
+
+# DIR_CID=bafybeig6ka5mlwkl4subqhaiatalkcleo4jgnr3hqwvpmsqfca27cijp3i # ./rootDir/
+# FILE_CID=bafkreialihlqnf5uwo4byh4n3cmwlntwqzxxs2fg5vanqdi3d7tb2l5xkm # ./rootDir/ą/ę/file-źł.txt
+# FILE_SIZE=34
+```

test/sharness/t0116-gateway-cache.sh

@@ -25,15 +25,16 @@ test_launch_ipfs_daemon_without_network
 # Caching of things like raw blocks, CARs, dag-json and dag-cbor
 # is tested in their respective suites.
 
+# Import test case
+# See the static fixtures in ./t0116-gateway-cache/
 test_expect_success "Add the test directory" '
-  mkdir -p root2/root3/root4 &&
-  echo "hello" > root2/root3/root4/index.html &&
-  ROOT1_CID=$(ipfs add -Qrw --cid-version 1 root2)
-  ROOT2_CID=$(ipfs resolve -r /ipfs/$ROOT1_CID/root2 | cut -d "/" -f3)
-  ROOT3_CID=$(ipfs resolve -r /ipfs/$ROOT1_CID/root2/root3 | cut -d "/" -f3)
-  ROOT4_CID=$(ipfs resolve -r /ipfs/$ROOT1_CID/root2/root3/root4 | cut -d "/" -f3)
-  FILE_CID=$(ipfs resolve -r /ipfs/$ROOT1_CID/root2/root3/root4/index.html | cut -d "/" -f3)
+  ipfs dag import ../t0116-gateway-cache/fixtures.car
 '
+ROOT1_CID=bafybeib3ffl2teiqdncv3mkz4r23b5ctrwkzrrhctdbne6iboayxuxk5ui # ./
+ROOT2_CID=bafybeih2w7hjocxjg6g2ku25hvmd53zj7og4txpby3vsusfefw5rrg5sii # ./root2
+ROOT3_CID=bafybeiawdvhmjcz65x5egzx4iukxc72hg4woks6v6fvgyupiyt3oczk5ja # ./root2/root3
+ROOT4_CID=bafybeifq2rzpqnqrsdupncmkmhs3ckxxjhuvdcbvydkgvch3ms24k5lo7q # ./root2/root3/root4
+FILE_CID=bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am # ./root2/root3/root4/index.html
 
 test_expect_success "Prepare IPNS unixfs content path for testing" '
   TEST_IPNS_ID=$(ipfs key gen --ipns-base=base36 --type=ed25519 cache_test_key | head -n1 | tr -d "\n")

test/sharness/t0116-gateway-cache/README.md

@@ -0,0 +1,31 @@
+# Dataset description/sources
+
+- fixtures.car
+  - raw CARv1
+
+generated with:
+
+```sh
+# using ipfs version 0.18.1
+mkdir -p root2/root3/root4 &&
+echo "hello" > root2/root3/root4/index.html &&
+ROOT1_CID=$(ipfs add -Qrw --cid-version 1 root2)
+ROOT2_CID=$(ipfs resolve -r /ipfs/$ROOT1_CID/root2 | cut -d "/" -f3)
+ROOT3_CID=$(ipfs resolve -r /ipfs/$ROOT1_CID/root2/root3 | cut -d "/" -f3)
+ROOT4_CID=$(ipfs resolve -r /ipfs/$ROOT1_CID/root2/root3/root4 | cut -d "/" -f3)
+FILE_CID=$(ipfs resolve -r /ipfs/$ROOT1_CID/root2/root3/root4/index.html | cut -d "/" -f3)
+
+echo ROOT1_CID=${ROOT1_CID}
+echo ROOT2_CID=${ROOT2_CID}
+echo ROOT3_CID=${ROOT3_CID}
+echo ROOT4_CID=${ROOT4_CID}
+echo FILE_CID=${FILE_CID}
+
+ipfs dag export ${ROOT1_CID} > ./fixtures.car
+
+# ROOT1_CID=bafybeib3ffl2teiqdncv3mkz4r23b5ctrwkzrrhctdbne6iboayxuxk5ui # ./
+# ROOT2_CID=bafybeih2w7hjocxjg6g2ku25hvmd53zj7og4txpby3vsusfefw5rrg5sii # ./root2
+# ROOT3_CID=bafybeiawdvhmjcz65x5egzx4iukxc72hg4woks6v6fvgyupiyt3oczk5ja # ./root2/root3
+# ROOT4_CID=bafybeifq2rzpqnqrsdupncmkmhs3ckxxjhuvdcbvydkgvch3ms24k5lo7q # ./root2/root3/root4
+# FILE_CID=bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am # ./root2/root3/root4/index.html
+```

test/sharness/t0117-gateway-block.sh

@@ -7,12 +7,13 @@ test_description="Test HTTP Gateway Raw Block (application/vnd.ipld.raw) Support
 test_init_ipfs
 test_launch_ipfs_daemon_without_network
 
-test_expect_success "Create text fixtures" '
-  mkdir -p dir &&
-  echo "hello application/vnd.ipld.raw" > dir/ascii.txt &&
-  ROOT_DIR_CID=$(ipfs add -Qrw --cid-version 1 dir) &&
-  FILE_CID=$(ipfs resolve -r /ipfs/$ROOT_DIR_CID/dir/ascii.txt | cut -d "/" -f3)
+# Import test case
+# See the static fixtures in ./t0117-gateway-block/
+test_expect_success "Add the dir test directory" '
+  ipfs dag import ../t0117-gateway-block/fixtures.car
 '
+ROOT_DIR_CID=bafybeie72edlprgtlwwctzljf6gkn2wnlrddqjbkxo3jomh4n7omwblxly # ./
+FILE_CID=bafkreihhpc5y2pqvl5rbe5uuyhqjouybfs3rvlmisccgzue2kkt5zq6upq # ./dir/ascii.txt
 
 # GET unixfs dir root block and compare it with `ipfs block get` output
 

test/sharness/t0117-gateway-block/README.md

@@ -0,0 +1,21 @@
+# Dataset description/sources
+
+- fixtures.car
+  - raw CARv1
+
+generated with:
+
+```sh
+# using ipfs version 0.18.1
+mkdir -p dir &&
+echo "hello application/vnd.ipld.raw" > dir/ascii.txt &&
+ROOT_DIR_CID=$(ipfs add -Qrw --cid-version 1 dir) &&
+FILE_CID=$(ipfs resolve -r /ipfs/$ROOT_DIR_CID/dir/ascii.txt | cut -d "/" -f3) &&
+ipfs dag export $ROOT_DIR_CID > fixtures.car
+
+echo ROOT_DIR_CID=${ROOT_DIR_CID} # ./
+echo FILE_CID=${FILE_CID} # ./dir/ascii.txt
+
+# ROOT_DIR_CID=bafybeie72edlprgtlwwctzljf6gkn2wnlrddqjbkxo3jomh4n7omwblxly # ./
+# FILE_CID=bafkreihhpc5y2pqvl5rbe5uuyhqjouybfs3rvlmisccgzue2kkt5zq6upq # ./dir/ascii.txt
+```

test/sharness/t0118-gateway-car.sh

@@ -11,15 +11,14 @@ test_launch_ipfs_daemon_without_network
 # but if we have a small file that fits into a single block, and export its CID
 # we will get a CAR that is a deterministic array of bytes.
 
-    test_expect_success "Create a deterministic CAR for testing" '
-    mkdir -p subdir &&
-    echo "hello application/vnd.ipld.car" > subdir/ascii.txt &&
-    ROOT_DIR_CID=$(ipfs add -Qrw --cid-version 1 subdir) &&
-    FILE_CID=$(ipfs resolve -r /ipfs/$ROOT_DIR_CID/subdir/ascii.txt | cut -d "/" -f3) &&
-    ipfs dag export $ROOT_DIR_CID > test-dag.car &&
-    ipfs dag export $FILE_CID > deterministic.car &&
-    purge_blockstore
-    '
+# Import test case
+# See the static fixtures in ./t0118-gateway-car/
+test_expect_success "Add the dir test directory" '
+    cp ../t0118-gateway-car/test-dag.car ./test-dag.car &&
+    cp ../t0118-gateway-car/deterministic.car ./deterministic.car
+'
+ROOT_DIR_CID=bafybeiefu3d7oytdumk5v7gn6s7whpornueaw7m7u46v2o6omsqcrhhkzi # ./
+FILE_CID=bafkreifkam6ns4aoolg3wedr4uzrs3kvq66p4pecirz6y2vlrngla62mxm # /subdir/ascii.txt
 
 # GET a reference DAG with dag-cbor+dag-pb+raw blocks as CAR
 

test/sharness/t0118-gateway-car/README.md

@@ -8,3 +8,23 @@
   - description of the contents and layout of the raw CAR, encoded in DAG-JSON
   - Source: https://ipld.io/specs/transport/car/fixture/carv1-basic/carv1-basic.json
 
+- test-dag.car + deterministic.car
+  - raw CARv1
+
+generated with:
+
+```sh
+# using ipfs version 0.18.1
+mkdir -p subdir &&
+echo "hello application/vnd.ipld.car" > subdir/ascii.txt &&
+ROOT_DIR_CID=$(ipfs add -Qrw --cid-version 1 subdir) &&
+FILE_CID=$(ipfs resolve -r /ipfs/$ROOT_DIR_CID/subdir/ascii.txt | cut -d "/" -f3) &&
+ipfs dag export $ROOT_DIR_CID > test-dag.car &&
+ipfs dag export $FILE_CID > deterministic.car &&
+
+echo ROOT_DIR_CID=${ROOT_DIR_CID} # ./
+echo FILE_CID=${FILE_CID} # /\subdir/ascii.txt
+
+# ROOT_DIR_CID=bafybeiefu3d7oytdumk5v7gn6s7whpornueaw7m7u46v2o6omsqcrhhkzi # ./
+# FILE_CID=bafkreifkam6ns4aoolg3wedr4uzrs3kvq66p4pecirz6y2vlrngla62mxm # /subdir/ascii.txt
+```

test/sharness/t0119-prometheus.sh

@@ -27,7 +27,7 @@ test_expect_success "filter metrics" '
 '
 
 test_expect_success "make sure metrics haven't changed" '
-  diff -u ../t0116-prometheus-data/prometheus_metrics filtered_metrics
+  diff -u ../t0119-prometheus-data/prometheus_metrics filtered_metrics
 '
 
 # Check what was added by enabling ResourceMgr.Enabled
@@ -50,11 +50,11 @@ test_kill_ipfs_daemon
 
 test_expect_success "filter metrics and find ones added by enabling ResourceMgr" '
   sed -ne "s/^\([a-z0-9_]\+\).*/\1/p" raw_metrics | LC_ALL=C sort > filtered_metrics &&
-  grep -v -x -f ../t0116-prometheus-data/prometheus_metrics filtered_metrics > rcmgr_metrics
+  grep -v -x -f ../t0119-prometheus-data/prometheus_metrics filtered_metrics > rcmgr_metrics
 '
 
 test_expect_success "make sure initial metrics added by setting ResourceMgr.Enabled haven't changed" '
-  diff -u ../t0116-prometheus-data/prometheus_metrics_added_by_enabling_rcmgr rcmgr_metrics
+  diff -u ../t0119-prometheus-data/prometheus_metrics_added_by_enabling_rcmgr rcmgr_metrics
 '
 
 test_done