Tutorial: Extract File from Blockchain

Example Case

We have uploaded the file hello.txt and this is the resulting Pigi Upload API Response:

{
  "pigi": {
    "version": 0,
    "algorithm": "aes-256-cbc:lzma:cbor",
    "encryptedSize": 96,
    "_algorithmIndex": 0,
    "_idHex": "03ba204e50d126e4674c005e04d82e84c21366780af1f43bd54a37816b6ab340"
  },
  "bitcoin": {
    "fee": 37,
    "size": 363,
    "txid": "af74641a2acaa40cc19244a8a8a4a62a1e6028ed35b7bb4982765b8be7cec011",
    "rawtx": "01000000014b797ea66f53ba08eda316f51d7ff3bee0d708f2d7772d4b9f99765af0f3eee8010000006a4730440220776d90939d1a5781a730749368ac8a56c5d50f6bd0f4a263c17f77a95df543360220756df6c9e4af4ecccba1c0bd85a447ba0e3ba12382877cf93ff547a4050133db412102c0e0e22fe226ad86eb98b1c59190d64870d893047c8e6eea2420de81f1fd577fffffffff020000000000000000a3006a086d696e74426c75654c96000003ba204e50d126e4674c005e04d82e84c21366780af1f43bd54a37816b6ab34000000060e2a315f97660897d9d3c25d0f7fd851a8a161b49ed7b8d0bba2a1c24a6204a7de4ff107d5f18a56a357d6dd4473ce598211f7a3c4b396c7660b697b5252c2c708cb294d3aca7cfca554cc3edbb9ba5b1e7c2dbf2a7b27c57966898788cb2bb373ba594939a586a36bce566c7756df3e4d7430000000000001976a9147e9eeb8c2f4bfecc6361b030dc0839fc5fbb401b88ac00000000",
    "explore": {
      "whatsonchain": "https://www.whatsonchain.com/tx/af74641a2acaa40cc19244a8a8a4a62a1e6028ed35b7bb4982765b8be7cec011",
      "blockchair": "https://blockchair.com/bitcoin-sv/transaction/af74641a2acaa40cc19244a8a8a4a62a1e6028ed35b7bb4982765b8be7cec011"
    }
  },
  "identifiers": {
    "file": [
      "https://test.api.next.kyrt.net/api/v1/pigi/f/03ba204e50d126e4674c005e04d82e84c21366780af1f43bd54a37816b6ab340?s=WmH5w6eCzNrYSAc80Hgtd2GZq8qwAtt6PS3Lct6yzzA0",
      "pigi://pigi.kyrt.net/f/03ba204e50d126e4674c005e04d82e84c21366780af1f43bd54a37816b6ab340?s=WmH5w6eCzNrYSAc80Hgtd2GZq8qwAtt6PS3Lct6yzzA0",
      "http://localhost:8777/api/v1/pigi/f/03ba204e50d126e4674c005e04d82e84c21366780af1f43bd54a37816b6ab340?s=WmH5w6eCzNrYSAc80Hgtd2GZq8qwAtt6PS3Lct6yzzA0"
    ],
    "raw": {
      "fileid": "03ba204e50d126e4674c005e04d82e84c21366780af1f43bd54a37816b6ab340",
      "secret": "WmH5w6eCzNrYSAc80Hgtd2GZq8qwAtt6PS3Lct6yzzA0",
      "txid": "af74641a2acaa40cc19244a8a8a4a62a1e6028ed35b7bb4982765b8be7cec011"
    },
    "transaction": [
      "https://test.api.next.kyrt.net/api/v1/pigi/t/af74641a2acaa40cc19244a8a8a4a62a1e6028ed35b7bb4982765b8be7cec011?s=WmH5w6eCzNrYSAc80Hgtd2GZq8qwAtt6PS3Lct6yzzA0",
      "pigi://pigi.kyrt.net/t/af74641a2acaa40cc19244a8a8a4a62a1e6028ed35b7bb4982765b8be7cec011?s=WmH5w6eCzNrYSAc80Hgtd2GZq8qwAtt6PS3Lct6yzzA0",
      "http://localhost:8777/api/v1/pigi/t/af74641a2acaa40cc19244a8a8a4a62a1e6028ed35b7bb4982765b8be7cec011?s=WmH5w6eCzNrYSAc80Hgtd2GZq8qwAtt6PS3Lct6yzzA0"
    ]
  },
  "info": {
    "file": [
      "https://test.api.next.kyrt.net/api/v1/pigi/fi/03ba204e50d126e4674c005e04d82e84c21366780af1f43bd54a37816b6ab340",
      "pigi://pigi.kyrt.net/fi/03ba204e50d126e4674c005e04d82e84c21366780af1f43bd54a37816b6ab340",
      "http://localhost:8777/api/v1/pigi/fi/03ba204e50d126e4674c005e04d82e84c21366780af1f43bd54a37816b6ab340"
    ],
    "transaction": [
      "https://test.api.next.kyrt.net/api/v1/pigi/ti/af74641a2acaa40cc19244a8a8a4a62a1e6028ed35b7bb4982765b8be7cec011",
      "pigi://pigi.kyrt.net/ti/af74641a2acaa40cc19244a8a8a4a62a1e6028ed35b7bb4982765b8be7cec011",
      "http://localhost:8777/api/v1/pigi/ti/af74641a2acaa40cc19244a8a8a4a62a1e6028ed35b7bb4982765b8be7cec011"
    ]
  }
}

Bitcoin Transactions in a Nutshell

A bitcon transaction contains INPUTS and OUTPUTS. See https://learnmeabitcoin.com/technical/transaction-data for more detail. The file that we just uploaded is stored the first OUTPUT

Get a Raw Transaction by TXID

It's possible to fetch the raw transaction from any txid by calling public APIs.

Popular API's for Bitcoin SV are:

• What's On Chain: https://api.whatsonchain.com/v1/bsv/main/tx/af74641a2acaa40cc19244a8a8a4a62a1e6028ed35b7bb4982765b8be7cec011/hex

• Blockchair: https://api.blockchair.com/bitcoin-sv/raw/transaction/af74641a2acaa40cc19244a8a8a4a62a1e6028ed35b7bb4982765b8be7cec011

Inspecting the Decoded Transaction

To increase our understanding of the transaction that we just created. Let's inspect its JSON-decoded form. A bitcoin transaction is often serialized in hexadecimal format. This hexadecimal format can be parsed and represented as a JSON object. Blockchair's API will return the JSON represenation of the transaction in the data.<txid>.decoded_raw_transaction field.

{
  "data": {
    "af74641a2acaa40cc19244a8a8a4a62a1e6028ed35b7bb4982765b8be7cec011": {
      "raw_transaction": "01000000014b797ea66f53ba08eda316f51d7ff3bee0d708f2d7772d4b9f99765af0f3eee8010000006a4730440220776d90939d1a5781a730749368ac8a56c5d50f6bd0f4a263c17f77a95df543360220756df6c9e4af4ecccba1c0bd85a447ba0e3ba12382877cf93ff547a4050133db412102c0e0e22fe226ad86eb98b1c59190d64870d893047c8e6eea2420de81f1fd577fffffffff020000000000000000a3006a086d696e74426c75654c96000003ba204e50d126e4674c005e04d82e84c21366780af1f43bd54a37816b6ab34000000060e2a315f97660897d9d3c25d0f7fd851a8a161b49ed7b8d0bba2a1c24a6204a7de4ff107d5f18a56a357d6dd4473ce598211f7a3c4b396c7660b697b5252c2c708cb294d3aca7cfca554cc3edbb9ba5b1e7c2dbf2a7b27c57966898788cb2bb373ba594939a586a36bce566c7756df3e4d7430000000000001976a9147e9eeb8c2f4bfecc6361b030dc0839fc5fbb401b88ac00000000",
      "decoded_raw_transaction": {
        "txid": "af74641a2acaa40cc19244a8a8a4a62a1e6028ed35b7bb4982765b8be7cec011",
        "hash": "af74641a2acaa40cc19244a8a8a4a62a1e6028ed35b7bb4982765b8be7cec011",
        "version": 1,
        "size": 363,
        "locktime": 0,
        "vin": [
          {
            "txid": "e8eef3f05a76999f4b2d77d7f208d7e0bef37f1df516a3ed08ba536fa67e794b",
            "vout": 1,
            "scriptSig": {
              "asm": "30440220776d90939d1a5781a730749368ac8a56c5d50f6bd0f4a263c17f77a95df543360220756df6c9e4af4ecccba1c0bd85a447ba0e3ba12382877cf93ff547a4050133db[ALL|FORKID] 02c0e0e22fe226ad86eb98b1c59190d64870d893047c8e6eea2420de81f1fd577f",
              "hex": "4730440220776d90939d1a5781a730749368ac8a56c5d50f6bd0f4a263c17f77a95df543360220756df6c9e4af4ecccba1c0bd85a447ba0e3ba12382877cf93ff547a4050133db412102c0e0e22fe226ad86eb98b1c59190d64870d893047c8e6eea2420de81f1fd577f"
            },
            "sequence": 4294967295
          }
        ],
        "vout": [
          {
            "value": 0,
            "n": 0,
            "scriptPubKey": {
              "asm": "0 OP_RETURN 6d696e74426c7565 000003ba204e50d126e4674c005e04d82e84c21366780af1f43bd54a37816b6ab34000000060e2a315f97660897d9d3c25d0f7fd851a8a161b49ed7b8d0bba2a1c24a6204a7de4ff107d5f18a56a357d6dd4473ce598211f7a3c4b396c7660b697b5252c2c708cb294d3aca7cfca554cc3edbb9ba5b1e7c2dbf2a7b27c57966898788cb2bb373ba594939a586a36bce566c7756df3e4",
              "hex": "006a086d696e74426c75654c96000003ba204e50d126e4674c005e04d82e84c21366780af1f43bd54a37816b6ab34000000060e2a315f97660897d9d3c25d0f7fd851a8a161b49ed7b8d0bba2a1c24a6204a7de4ff107d5f18a56a357d6dd4473ce598211f7a3c4b396c7660b697b5252c2c708cb294d3aca7cfca554cc3edbb9ba5b1e7c2dbf2a7b27c57966898788cb2bb373ba594939a586a36bce566c7756df3e4",
              "type": "nulldata"
            }
          },
          {
            "value": 0.00017367,
            "n": 1,
            "scriptPubKey": {
              "asm": "OP_DUP OP_HASH160 7e9eeb8c2f4bfecc6361b030dc0839fc5fbb401b OP_EQUALVERIFY OP_CHECKSIG",
              "hex": "76a9147e9eeb8c2f4bfecc6361b030dc0839fc5fbb401b88ac",
              "reqSigs": 1,
              "type": "pubkeyhash",
              "addresses": ["1CYWX8jRhjkuitviPgP9vQ2Fs6tuuQ9tGD"]
            }
          }
        ],
        "hex": "01000000014b797ea66f53ba08eda316f51d7ff3bee0d708f2d7772d4b9f99765af0f3eee8010000006a4730440220776d90939d1a5781a730749368ac8a56c5d50f6bd0f4a263c17f77a95df543360220756df6c9e4af4ecccba1c0bd85a447ba0e3ba12382877cf93ff547a4050133db412102c0e0e22fe226ad86eb98b1c59190d64870d893047c8e6eea2420de81f1fd577fffffffff020000000000000000a3006a086d696e74426c75654c96000003ba204e50d126e4674c005e04d82e84c21366780af1f43bd54a37816b6ab34000000060e2a315f97660897d9d3c25d0f7fd851a8a161b49ed7b8d0bba2a1c24a6204a7de4ff107d5f18a56a357d6dd4473ce598211f7a3c4b396c7660b697b5252c2c708cb294d3aca7cfca554cc3edbb9ba5b1e7c2dbf2a7b27c57966898788cb2bb373ba594939a586a36bce566c7756df3e4d7430000000000001976a9147e9eeb8c2f4bfecc6361b030dc0839fc5fbb401b88ac00000000"
      }
    }
  },
  "context": {
    "code": 200,
    "source": "R",
    "results": 1,
    "state": 710073,
    "market_price_usd": 172.04,
    "cache": {
      "live": true,
      "duration": 60,
      "since": "2021-10-21 15:18:24",
      "until": "2021-10-21 15:19:24",
      "time": null
    },
    "api": {
      "version": "2.0.88",
      "last_major_update": "2021-07-19 00:00:00",
      "next_major_update": null,
      "documentation": "https://blockchair.com/api/docs",
      "notice": ":)"
    },
    "server": "BSV0",
    "time": 0.18567895889282227,
    "render_time": 0.9338440895080566,
    "full_time": 1.119523048400879,
    "request_cost": 1
  }
}

Inspecting the First Output

Take a closer look at decoded_raw_transaction.vout[0].scriptPubKey.asm. This is a Bitcoin Script and it contains an assembly language that is executed by miners in order to verify that a specific output can be spent. In this case, the output is unspendable and simply contains data. Let's dissect this script:

1. 0

2. OP_RETURN

3. 6d696e74426c7565

4. 000003ba204e50d126e4674c005e04d82e84c21366780af1f43bd54a37816b6ab34000000060e2a315f97660897d9d3c25d0f7fd851a8a161b49ed7b8d0bba2a1c24a6204a7de4ff107d5f18a56a357d6dd4473ce598211f7a3c4b396c7660b697b5252c2c708cb294d3aca7cfca554cc3edbb9ba5b1e7c2dbf2a7b27c57966898788cb2bb373ba594939a586a36bce566c7756df3e4

Extracting the File (pseudocode)

Part 1: Parsing

Now that we understand how a Pigi binary blob is stored within a transaction, lets parse the Pigi binary blob and decrypt its contents.

1. Read the first byte (version)

2. Read the second byte (algorithm)

3. Read the next 32 bytes (file id)

4. Read the next 4 bytes (length)

5. Read the next 16 bytes (IV)

6. Read the next encryptedSize bytes (encryptedData)

Part 2: Decrypting AES-256-CBC

The upload reponse returned a secret key inside the identifiers.raw.secret field. In our case: WmH5w6eCzNrYSAc80Hgtd2GZq8qwAtt6PS3Lct6yzzA0

This is a base62-encoded representation of a 32 byte secret key. To convert the base62 key to binary, use any base62 library and use the following alphabet:

'0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz'

secretKey = base62.decode('0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz');

Now that the key is in binary form. We can decrypt encryptedData as follows:

decryptedData = AES256CBC - DECRYPT(encryptedData, iv, secret);

Part 3: Decompressing

The decrypted data is an LZMA compressed binary blob. We must now decompress it

decompressedData = LZMA - DECOMPRESS(decryptedData);

Part 4: Decoding CBOR

The decompressed data contains a CBOR encoded array. We must use CBOR to decode this decompressed data. After we have CBOR decoded the decompressed data, we have access to the filename, file type and file contents.

cborDecodedData = CBOR-DECODE(decompressedData)

filename = cborDecodedData[0]
filetype = cborDecodedData[1]
filecontents = cborDecodedData[2]

Full JavaScript Example with Code Annotations

/**
 * This file contains helper functions to assemble and parse a Pigi binary message
 */

import crypto from 'crypto';
import r from 'restructure';
import base62 from '@fry/base62';
import cbor from 'cbor';

// eslint-disable-next-line @typescript-eslint/no-var-requires
const lzma = require('lzma-native');

export const Algorithms = ['aes-256-cbc:lzma:cbor'];

export const PigiStruct = new r.VersionedStruct(r.uint8, {
  0: {
    algorithm: new r.Enum(r.uint8, Algorithms),
    originalFileHash: new r.Buffer(32),
    encryptedSize: r.uint32be,
    iv: new r.Buffer(16),
    encryptedData: new r.Buffer('encryptedSize'),
  },
});

export interface Pigi {
  version: number;
  algorithm: string;
  originalFileHash: Buffer;
  encryptedSize: number;
  iv: Buffer;
  encryptedData: Buffer;
  _opReturnChunk: Buffer;
  _secret: Buffer;
  _secretHex: string;
  _secretBase62: string;
  _id: Buffer;
  _idHex: string;
  _idBase62: string;
  _algorithmIndex: number;
  _mimetype: string;
  _filename: string;
  _plainData: Buffer;
  _originalSize: number;
}

// This is the request handler to look up a file by txid and secret
export async function lookupByTxid(req: express.Request, res: express.Response): Promise<void> {
  try {
    // Get the txid from the request parameter
    const txid = req.params.txid;
    // Get the base62 secet from the query parameter
    const s = req.query.s;

    // Decode the base62 secret to binary
    const secret = base62.decode(s).slice(0, 32);
    // Fetch the raw transaction from WhatsOnChain
    const rawtx = await axios.get(`https://api.whatsonchain.com/v1/bsv/main/tx/${txid}/hex`).then((res) => res.data);

    // Parse the raw transaction
    const tx = new bsv.Transaction(rawtx);
    // Take the first output, and of that, the last script chunk (pigi binary blob)
    const pigiBuffer = tx.outputs[0].script.chunks[tx.outputs[0].script.chunks.length - 1].buf;

    // Decode the pigi binary blob (function is defined below)
    const p = await decodeOpReturn(pigiBuffer, secret);

    setDownloadResponseHeaders(p, res);
    res.setHeader('Pigi-Bitcoin-TxId', tx.hash);

    // The raw file contents is now in p._plainData
    res.send(p._plainData);
  } catch (e) {
    res.status(404).json({ error: e.message, stack: e.stack });
  }
}

export async function decodeOpReturn(opReturnChunk: Buffer, secret: Buffer): Promise<Pigi> {
  // Parse the pigi binary blob into its fields.
  const stream = new r.DecodeStream(opReturnChunk);
  const pigi = PigiStruct.decode(stream);

  const cipher = pigi.algorithm.split(':')[0]; // aes-256-cbc
  const iv = pigi.iv;

  // decrypt
  const decipher = crypto.createDecipheriv(cipher, secret, iv);
  let decrypt = decipher.update(pigi.encryptedData);
  decrypt = Buffer.concat([decrypt, decipher.final()]);

  // decompress
  const decompressed = await lzma.decompress(decrypt);

  // cbor decode
  const d = cbor.decode(decompressed);

  const filename = d[0];
  const mimetype = d[1];
  const filebuf = d[2];

  return {
    version: pigi.version,
    algorithm: pigi.algorithm,
    originalFileHash: pigi.originalFileHash,
    encryptedSize: pigi.encryptedSize,
    iv: pigi.iv,
    encryptedData: pigi.encryptedData,
    _algorithmIndex: 0,
    _filename: filename,
    _id: pigi.hashOriginalFile,
    _idBase62: base62.encode(pigi.hashOriginalFile),
    _idHex: pigi.originalFileHash.toString('hex'),
    _secret: secret,
    _secretHex: secret.toString('hex'),
    _secretBase62: base62.encode(secret),
    _mimetype: mimetype,
    _opReturnChunk: opReturnChunk,
    _plainData: filebuf,
    _originalSize: filebuf.length,
  };
}