Skip to content

Add support for retrieval of PDP pieces without CDN enabled #23

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 3 commits into from
Closed

Add support for retrieval of PDP pieces without CDN enabled #23

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
9353502
Enable retrieval of PDP deals without CDN flag enabled
pyropy Aug 19, 2025
5dcc2fe
Make withCDN flag boolean, not boolean or undefined
pyropy Aug 20, 2025
2063a7d
Merge branch 'main' into feat/enable-retrieval-of-non-cdn-deals
juliangruber Sep 11, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions bin/bot.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,7 @@ await Promise.all([
botLocation: FLY_REGION,
CDN_HOSTNAME,
FROM_DATA_SET_ID: BigInt(FROM_DATA_SET_ID),
withCDN: true,
})
console.log('\n')
await setTimeout(Number(DELAY))
Expand All @@ -71,6 +72,7 @@ await Promise.all([
botLocation: FLY_REGION,
CDN_HOSTNAME,
FROM_DATA_SET_ID: BigInt(FROM_DATA_SET_ID),
withCDN: true,
})
console.log('\n')
await setTimeout(Number(30_000)) // block time
Expand Down
89 changes: 59 additions & 30 deletions index.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -133,6 +133,7 @@ export const serviceProviderRegistryAbi = [
* @param {string} args.pieceCid
* @param {BigInt} args.dataSetId
* @param {BigInt} args.pieceId
* @param {boolean} [args.withCDN=true] Default is `true`
* @param {boolean} [args.retryOn404=true] Default is `true`
* @param {number} [args.retryDelayMs=10_000] Default is `10_000`
* @returns {Promise<void>}
Expand All @@ -147,10 +148,29 @@ async function testRetrieval({
pieceCid,
dataSetId,
pieceId,
withCDN = true,
retryOn404 = true,
retryDelayMs = 10_000,
}) {
const url = `https://${clientAddress}.${CDN_HOSTNAME}/${pieceCid}`
let baseUrl
if (withCDN) {
baseUrl = `https://${clientAddress}.${CDN_HOSTNAME}`
} else {
baseUrl = await maybeGetResolvedDataSetRetrievalUrl({
pdpVerifier,
fwssStateView,
serviceProviderRegistry,
dataSetId,
})
if (!baseUrl) {
console.error(
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
console.error(
throw new Error(

Otherwise withCDN=false tests can silently fail

`Cannot resolve retrieval URL for data set ${dataSetId} piece ${pieceId} from ${botLocation ?? '<dev>'}`,
)
return
}
}

const url = `${baseUrl}/${pieceCid}`
console.log('Fetching', url)
const res = await fetch(url)
console.log('-> Status code:', res.status)
Expand Down Expand Up @@ -187,7 +207,7 @@ async function testRetrieval({
pdpVerifier,
fwssStateView,
serviceProviderRegistry,
dataSetIdHeaderValue,
dataSetId: dataSetIdHeaderValue,
})

console.error(
Expand Down Expand Up @@ -217,33 +237,30 @@ async function testRetrieval({
* @param {PdpVerifier} args.pdpVerifier
* @param {FilecoinWarmStorageServiceStateView} args.fwssStateView
* @param {ServiceProviderRegistry} args.serviceProviderRegistry
* @param {string | null} args.dataSetIdHeaderValue
* @param {bigint | string | null} args.dataSetId
* @returns {Promise<string | undefined>} The piece retrieval URL
*/
async function maybeGetResolvedDataSetRetrievalUrl({
pdpVerifier,
fwssStateView,
serviceProviderRegistry,
dataSetIdHeaderValue,
dataSetId,
}) {
if (dataSetIdHeaderValue === null || dataSetIdHeaderValue === '') {
if (dataSetId === null || dataSetId === '') {
return undefined
}

let dataSetId
let dataSetIdValue
try {
dataSetId = BigInt(dataSetIdHeaderValue)
dataSetIdValue =
typeof dataSetId === 'bigint' ? dataSetId : BigInt(dataSetId)
} catch (err) {
console.warn(
'FilCDN reported invalid DataSetID %j: %s',
dataSetIdHeaderValue,
err,
)
console.warn('FilCDN reported invalid data set ID %j: %s', dataSetId, err)
return undefined
}

try {
const [dataSetOwner] = await pdpVerifier.getDataSetOwner(dataSetId)
const [dataSetOwner] = await pdpVerifier.getDataSetOwner(dataSetIdValue)
const providerId =
await serviceProviderRegistry.getProviderIdByAddress(dataSetOwner)

Expand Down Expand Up @@ -290,6 +307,7 @@ async function maybeGetResolvedDataSetRetrievalUrl({
* @param {string} [args.botLocation] Fly region where the bot is running
* @param {string} args.CDN_HOSTNAME
* @param {BigInt} args.FROM_DATA_SET_ID
* @param {boolean} [args.withCDN=true] Default is `true`
*/

export async function sampleRetrieval({
Expand All @@ -299,14 +317,15 @@ export async function sampleRetrieval({
botLocation = '<dev>',
CDN_HOSTNAME,
FROM_DATA_SET_ID,
withCDN = true,
}) {
const { pieceCid, dataSetId, pieceId, clientAddress } =
await pickRandomFileWithCDN({
pdpVerifier,
fwssStateView,
serviceProviderRegistry,
FROM_DATA_SET_ID,
})
const { pieceCid, dataSetId, pieceId, clientAddress } = await pickRandomFile({
pdpVerifier,
fwssStateView,
serviceProviderRegistry,
FROM_DATA_SET_ID,
withCDN,
})

await testRetrieval({
pdpVerifier,
Expand All @@ -318,6 +337,7 @@ export async function sampleRetrieval({
pieceCid,
dataSetId,
pieceId,
withCDN,
})
}

Expand All @@ -327,6 +347,7 @@ export async function sampleRetrieval({
* @param {FilecoinWarmStorageServiceStateView} args.fwssStateView
* @param {ServiceProviderRegistry} args.serviceProviderRegistry
* @param {BigInt} args.FROM_DATA_SET_ID
* @param {boolean} [args.withCDN=true] Default is `true`
* @returns {Promise<{
* pieceCid: string
* dataSetId: BigInt
Expand All @@ -335,11 +356,12 @@ export async function sampleRetrieval({
* }>}
* The CommP CID of the file.
*/
async function pickRandomFileWithCDN({
async function pickRandomFile({
pdpVerifier,
fwssStateView,
serviceProviderRegistry,
FROM_DATA_SET_ID,
withCDN = true,
}) {
// Cache state query responses to speed up the sampling algorithm.
/** @type {Map<BigInt, DataSetInfo>} */
Expand Down Expand Up @@ -373,16 +395,15 @@ async function pickRandomFileWithCDN({
(await fwssStateView.getDataSet(dataSetId))
cachedDataSetsInfo.set(dataSetId, dataSet)
const { payer: clientAddress, payee: providerAddress } = dataSet
const { exists: withCDN } = await fwssStateView.getDataSetMetadata(
const { exists: dataSetWithCDN } = await fwssStateView.getDataSetMetadata(
dataSetId,
'withCDN',
)

if (!withCDN) {
if (withCDN !== dataSetWithCDN) {
console.log(
'data set does not pay for CDN, restarting the sampling algorithm',
'Data set CDN status does not match, restarting the sampling algorithm',
)
continue
}

const providerId =
Expand Down Expand Up @@ -459,6 +480,7 @@ async function pickRandomFileWithCDN({
* @param {string} [args.botLocation] Fly region where the bot is running
* @param {string} args.CDN_HOSTNAME
* @param {BigInt} args.FROM_DATA_SET_ID
* @param {boolean} [args.withCDN=true] Default is `true`
*/

export async function testLatestRetrievablePiece({
Expand All @@ -468,13 +490,15 @@ export async function testLatestRetrievablePiece({
botLocation,
CDN_HOSTNAME,
FROM_DATA_SET_ID,
withCDN = true,
}) {
const { pieceCid, dataSetId, pieceId, clientAddress } =
await getMostRecentFileWithCDN({
await getMostRecentFile({
pdpVerifier,
fwssStateView,
serviceProviderRegistry,
FROM_DATA_SET_ID,
withCDN,
})

await testRetrieval({
Expand All @@ -487,6 +511,7 @@ export async function testLatestRetrievablePiece({
pieceCid,
dataSetId,
pieceId,
withCDN,
})
}

Expand All @@ -496,6 +521,7 @@ export async function testLatestRetrievablePiece({
* @param {FilecoinWarmStorageServiceStateView} args.fwssStateView
* @param {ServiceProviderRegistry} args.serviceProviderRegistry
* @param {BigInt} args.FROM_DATA_SET_ID
* @param {boolean} [args.withCDN=true] Default is `true`
* @returns {Promise<{
* pieceCid: string
* dataSetId: BigInt
Expand All @@ -504,11 +530,12 @@ export async function testLatestRetrievablePiece({
* }>}
* The CommP CID of the file.
*/
async function getMostRecentFileWithCDN({
async function getMostRecentFile({
pdpVerifier,
fwssStateView,
serviceProviderRegistry,
FROM_DATA_SET_ID,
withCDN = true,
}) {
for (
let dataSetId = (await pdpVerifier.getNextDataSetId()) - 1n;
Expand All @@ -527,13 +554,15 @@ async function getMostRecentFileWithCDN({
await fwssStateView.getDataSet(dataSetId)

// If `withCDN` metadata key is present it means the data set pays for CDN
const { exists: withCDN } = await fwssStateView.getDataSetMetadata(
const { exists: dataSetWithCDN } = await fwssStateView.getDataSetMetadata(
dataSetId,
'withCDN',
)

if (!withCDN) {
console.log('data set does not pay for CDN')
if (withCDN !== dataSetWithCDN) {
console.log(
`Data set has withCDN: ${dataSetWithCDN} but we are looking for withCDN: ${withCDN}`,
)
continue
}

Expand Down