diff --git a/core/corehttp/gateway_handler.go b/core/corehttp/gateway_handler.go index 854d33bd576..ffa814f880b 100644 --- a/core/corehttp/gateway_handler.go +++ b/core/corehttp/gateway_handler.go @@ -327,6 +327,9 @@ func (i *gatewayHandler) getOrHeadHandler(w http.ResponseWriter, r *http.Request // Support custom response formats passed via ?format or Accept HTTP header switch contentType := getExplicitContentType(r); contentType { + case "": + // nothing we should special-case, skip + break case "application/vnd.ipld.raw": logger.Debugw("serving raw block", "path", parsedPath) i.serveRawBlock(w, r, resolvedPath.Cid(), parsedPath) diff --git a/core/corehttp/gateway_handler_car.go b/core/corehttp/gateway_handler_car.go index 30a90c5c801..812b413c340 100644 --- a/core/corehttp/gateway_handler_car.go +++ b/core/corehttp/gateway_handler_car.go @@ -20,32 +20,31 @@ func (i *gatewayHandler) serveCar(w http.ResponseWriter, r *http.Request, rootCi name := rootCid.String() + ".car" setContentDispositionHeader(w, name, "attachment") - // Set remaining headers - /* TODO modtime := addCacheControlHeaders(w, r, contentPath, rootCid) - - how does cache-control look like, given car can fail mid-stream? - - we don't want clients to cache partial/interrupted CAR - - we may document that client should verify that all blocks were dowloaded, - or we may leverage content-length to hint something went wrong - */ - - /* TODO: content-length (so user agents show % of remaining download) - - introduce max-car-size limit in go-ipfs-config and pre-compute CAR first, and then get size and use lazySeeker? - - are we able to provide length for Unixfs DAGs? (CumulativeSize+CARv0 header+envelopes) - */ + // Weak Etag W/ because we can't guarantee byte-for-byte identical responses + // (CAR is streamed, blocks arrive from datastore in non-deterministic order) + w.Header().Set("Etag", `W/"`+rootCid.String()+`.car"`) + + // Explicit Cache-Control to ensure fresh stream on retry. + // CAR stream could be interrupted, and client should be able to resume and get full response, not the truncated one + w.Header().Set("Cache-Control", "no-cache, no-transform") w.Header().Set("Content-Type", "application/vnd.ipld.car; version=1") w.Header().Set("X-Content-Type-Options", "nosniff") // no funny business in the browsers :^) // Same go-car settings as dag.export command store := dagStore{dag: i.api.Dag(), ctx: ctx} + + // TODO: support selectors passed as request param: https://github.com/ipfs/go-ipfs/issues/8769 dag := gocar.Dag{Root: rootCid, Selector: selectorparse.CommonSelector_ExploreAllRecursively} car := gocar.NewSelectiveCar(ctx, store, []gocar.Dag{dag}, gocar.TraverseLinksOnlyOnce()) - w.WriteHeader(http.StatusOK) - if err := car.Write(w); err != nil { - // TODO: can we do any error handling here? - // TODO: idea: add best-effort proxy reader which will set http.StatusOK only if the first block is yielded correctly + // We return error as a trailer, however it is not something browsers can access + // (https://github.com/mdn/browser-compat-data/issues/14703) + // Due to this, we suggest client always verify that + // the received CAR stream response is matching requested DAG selector + w.Header().Set("X-Stream-Error", err.Error()) + return } } diff --git a/test/sharness/lib/test-lib.sh b/test/sharness/lib/test-lib.sh index a68c5d9737b..38f12a0250c 100644 --- a/test/sharness/lib/test-lib.sh +++ b/test/sharness/lib/test-lib.sh @@ -520,3 +520,16 @@ findprovs_expect() { test_cmp findprovsOut expected ' } + +purge_blockstore() { + ipfs pin ls --quiet --type=recursive | ipfs pin rm &>/dev/null + ipfs repo gc --silent &>/dev/null + + test_expect_success "pinlist empty" ' + [[ -z "$( ipfs pin ls )" ]] + ' + test_expect_success "nothing left to gc" ' + [[ -z "$( ipfs repo gc )" ]] + ' +} + diff --git a/test/sharness/t0117-gateway-block.sh b/test/sharness/t0117-gateway-block.sh index b04d4ac1762..3ce3f6a6f0a 100755 --- a/test/sharness/t0117-gateway-block.sh +++ b/test/sharness/t0117-gateway-block.sh @@ -5,12 +5,12 @@ test_description="Test HTTP Gateway Raw Block (application/vnd.ipld.raw) Support . lib/test-lib.sh test_init_ipfs -test_launch_ipfs_daemon +test_launch_ipfs_daemon_without_network test_expect_success "Create text fixtures" ' mkdir -p dir && - echo "hello" > dir/ascii.txt && - ROOT_DIR_CID=$(ipfs add -Qrw --cid-version 1 dir) + echo "hello application/vnd.ipld.raw" > dir/ascii.txt && + ROOT_DIR_CID=$(ipfs add -Qrw --cid-version 1 dir) && FILE_CID=$(ipfs resolve -r /ipfs/$ROOT_DIR_CID/dir/ascii.txt | cut -d "/" -f3) ' diff --git a/test/sharness/t0118-gateway-car.sh b/test/sharness/t0118-gateway-car.sh new file mode 100755 index 00000000000..fab9d37073c --- /dev/null +++ b/test/sharness/t0118-gateway-car.sh @@ -0,0 +1,97 @@ +#!/usr/bin/env bash + +test_description="Test HTTP Gateway CAR (application/vnd.ipld.car) Support" + +. lib/test-lib.sh + +test_init_ipfs +test_launch_ipfs_daemon_without_network + +# CAR stream is not deterministic, as blocks can arrive in random order, +# but if we have a small file that fits into a single block, and export its CID +# we will get a CAR that is a deterministic array of bytes. + +test_expect_success "Create a deterministic CAR for testing" ' + mkdir -p subdir && + echo "hello application/vnd.ipld.car" > subdir/ascii.txt && + ROOT_DIR_CID=$(ipfs add -Qrw --cid-version 1 subdir) && + FILE_CID=$(ipfs resolve -r /ipfs/$ROOT_DIR_CID/subdir/ascii.txt | cut -d "/" -f3) && + ipfs dag export $ROOT_DIR_CID > test-dag.car && + ipfs dag export $FILE_CID > deterministic.car && + purge_blockstore +' + +# GET unixfs file as CAR +# (by using a single file we ensure deterministic result that can be compared byte-for-byte) + + test_expect_success "GET with format=car param returns a CARv1 stream" ' + ipfs dag import test-dag.car && + curl -sX GET "http://127.0.0.1:$GWAY_PORT/ipfs/$ROOT_DIR_CID/subdir/ascii.txt?format=car" -o gateway-param.car && + test_cmp deterministic.car gateway-param.car + ' + + test_expect_success "GET for application/vnd.ipld.car returns a CARv1 stream" ' + ipfs dag import test-dag.car && + curl -sX GET -H "Accept: application/vnd.ipld.car" "http://127.0.0.1:$GWAY_PORT/ipfs/$ROOT_DIR_CID/subdir/ascii.txt" -o gateway-header.car && + test_cmp deterministic.car gateway-header.car + ' + + # explicit version=1 + test_expect_success "GET for application/vnd.ipld.raw version=1 returns a CARv1 stream" ' + ipfs dag import test-dag.car && + curl -sX GET -H "Accept: application/vnd.ipld.car; version=1" "http://127.0.0.1:$GWAY_PORT/ipfs/$ROOT_DIR_CID/subdir/ascii.txt" -o gateway-header-v1.car && + test_cmp deterministic.car gateway-header-v1.car + ' + +# GET unixfs directory as a CAR with DAG and some selector + + # TODO: this is basic test for "full" selector, we will add support for custom ones in https://github.com/ipfs/go-ipfs/issues/8769 + test_expect_success "GET for application/vnd.ipld.car with unixfs dir returns a CARv1 stream with full DAG" ' + ipfs dag import test-dag.car && + curl -sX GET -H "Accept: application/vnd.ipld.car" "http://127.0.0.1:$GWAY_PORT/ipfs/$ROOT_DIR_CID" -o gateway-dir.car && + purge_blockstore && + ipfs dag import gateway-dir.car && + ipfs dag stat --offline $ROOT_DIR_CID + ' + +# Make sure expected HTTP headers are returned with the block bytes + + test_expect_success "GET response for application/vnd.ipld.car has expected Content-Type" ' + ipfs dag import test-dag.car && + curl -svX GET -H "Accept: application/vnd.ipld.car" "http://127.0.0.1:$GWAY_PORT/ipfs/$ROOT_DIR_CID/subdir/ascii.txt" >/dev/null 2>curl_output && + cat curl_output && + grep "< Content-Type: application/vnd.ipld.car; version=1" curl_output + ' + + # CAR is streamed, gateway may not have the entire thing, unable to calculate total size + test_expect_success "GET response for application/vnd.ipld.car includes no Content-Length" ' + grep -qv "< Content-Length:" curl_output + ' + + test_expect_success "GET response for application/vnd.ipld.car includes Content-Disposition" ' + grep "< Content-Disposition: attachment\; filename=\"${FILE_CID}.car\"" curl_output + ' + + test_expect_success "GET response for application/vnd.ipld.car includes nosniff hint" ' + grep "< X-Content-Type-Options: nosniff" curl_output + ' + +# Cache control HTTP headers + + test_expect_success "GET response for application/vnd.ipld.car includes a weak Etag" ' + grep "< Etag: W/\"${FILE_CID}.car\"" curl_output + ' + + # (basic checks, detailed behavior for some fields is tested in t0116-gateway-cache.sh) + test_expect_success "GET response for application/vnd.ipld.car includes X-Ipfs-Path and X-Ipfs-Roots" ' + grep "< X-Ipfs-Path" curl_output && + grep "< X-Ipfs-Roots" curl_output + ' + + test_expect_success "GET response for application/vnd.ipld.raw includes expected Cache-Control" ' + grep "< Cache-Control: no-cache, no-transform" curl_output + ' + +test_kill_ipfs_daemon + +test_done