Metadata and Data - Upstash Documentation

Metadata

Metadata feature allows you to store context with your vectors to make a connection. There can be a couple of uses of this:

You can put the source of the vector in the metadata to use in your application from the query response.
You can put some metadata to further filter the results upon the query.

You can set metadata with your vector as follows:

Python
JavaScript
Go
PHP
curl

from upstash_vector import Index

index = Index(
    url="UPSTASH_VECTOR_REST_URL",
    token="UPSTASH_VECTOR_REST_TOKEN",
)

index.upsert(
    [("id-0", [0.9215, 0.3897]), {"url": "https://imgur.com/z9AVZLb"}],
)

import { Index } from "@upstash/vector"

const index = new Index({
  url: "UPSTASH_VECTOR_REST_URL",
  token: "UPSTASH_VECTOR_REST_TOKEN",
})

await index.upsert({
  "id": "id-0",
  vector: [0.9215, 0.3897],
  metadata: {
    url: "https://imgur.com/z9AVZLb",
  },
})

package main

import (
	"github.com/upstash/vector-go"
)

func main() {
	index := vector.NewIndex("UPSTASH_VECTOR_REST_URL", "UPSTASH_VECTOR_REST_TOKEN")

	index.Upsert(vector.Upsert{
		Id:       "id-0",
		Vector:   []float32{0.9215, 0.3897},
		Metadata: map[string]any{"url": "https://imgur.com/z9AVZLb"},
	})
}

use Upstash\Vector\Index;
use Upstash\Vector\VectorUpsert;

use function Upstash\Vector\createRandomVector;

$index = new Index(
  url: 'UPSTASH_VECTOR_REST_URL',
  token: 'UPSTASH_VECTOR_REST_TOKEN',
);

$index->upsert(new VectorUpsert(
  id: 'id-0',
  vector: createRandomVector(384),
  metadata: [
    'url' => "https://imgur.com/z9AVZLb",
  ],
));

curl $UPSTASH_VECTOR_REST_URL/upsert \
  -X POST \
  -H "Authorization: Bearer $UPSTASH_VECTOR_REST_TOKEN" \
  -d '{
   "id":"id-0",
   "vector":[0.9215,0.3897],
   "metadata":{
      "url":"https://imgur.com/z9AVZLb"
   }
}'

When you do a query or fetch, you can opt-in to retrieve the metadata as follows:

Query Example

Python
JavaScript
Go
PHP
curl

from upstash_vector import Index

index = Index(
    url="UPSTASH_VECTOR_REST_URL",
    token="UPSTASH_VECTOR_REST_TOKEN",
)

index.query(
    [0.9215, 0.3897],
    top_k=5,
    include_metadata=True,
)

import { Index } from "@upstash/vector"

const index = new Index({
  url: "UPSTASH_VECTOR_REST_URL",
  token: "UPSTASH_VECTOR_REST_TOKEN",
})

await index.query({
  vector: [0.9215, 0.3897],
  topK: 5,
  includeMetadata: true,
})

package main

import (
	"github.com/upstash/vector-go"
)

func main() {
	index := vector.NewIndex("UPSTASH_VECTOR_REST_URL", "UPSTASH_VECTOR_REST_TOKEN")

	index.Query(vector.Query{
		Vector:          []float32{0.9215, 0.3897},
		TopK:            5,
		IncludeMetadata: true,
	})
}

use Upstash\Vector\Index;
use Upstash\Vector\VectorQuery;

$index = new Index(
  url: 'UPSTASH_VECTOR_REST_URL',
  token: 'UPSTASH_VECTOR_REST_TOKEN',
);

$index->query(new VectorQuery(
  vector: [0.9215, 0.3897],
  topK: 5,
  includeMetadata: true,
));

curl $UPSTASH_VECTOR_REST_URL/query \
  -H "Authorization: Bearer $UPSTASH_VECTOR_REST_TOKEN" \
  -d '{
   "vector":[0.9215,0.3897],
   "topK" : 5,
   "includeMetadata": true
}'

{
  "result": [
    {
      "id": "id-0",
      "score": 1,
      "metadata": {
        "url": "https://imgur.com/z9AVZLb"
      }
    },
    {
      "id": "id-3",
      "score": 0.99961007,
      "metadata": {
        "url": "https://imgur.com/zfOPmnI"
      }
    }
  ]
}

Also, you can filter the results further by providing a metadata filter:

Python
JavaScript
Go
PHP
curl

from upstash_vector import Index

index = Index(
    url="UPSTASH_VECTOR_REST_URL",
    token="UPSTASH_VECTOR_REST_TOKEN",
)

index.query(
    [0.9215, 0.3897],
    top_k=5,
    include_metadata=True,
    filter="url GLOB '*imgur.com*'",
)

import { Index } from "@upstash/vector"

const index = new Index({
  url: "UPSTASH_VECTOR_REST_URL",
  token: "UPSTASH_VECTOR_REST_TOKEN",
})

await index.query({
  vector: [0.9215, 0.3897],
  topK: 5,
  includeMetadata: true,
  filter: "url GLOB '*imgur.com*'",
})

package main

import (
	"github.com/upstash/vector-go"
)

func main() {
	index := vector.NewIndex("UPSTASH_VECTOR_REST_URL", "UPSTASH_VECTOR_REST_TOKEN")

	index.Query(vector.Query{
		Vector:          []float32{0.9215, 0.3897},
		TopK:            5,
		IncludeMetadata: true,
		Filter:          "url GLOB '*imgur.com*'",
	})
}

use Upstash\Vector\Index;
use Upstash\Vector\VectorQuery;

$index = new Index(
  url: 'UPSTASH_VECTOR_REST_URL',
  token: 'UPSTASH_VECTOR_REST_TOKEN',
);

$index->query(new VectorQuery(
  vector: [0.9215, 0.3897],
  topK: 5,
  includeMetadata: true,
  filter: "url GLOB '*imgur.com*'",
));

curl $UPSTASH_VECTOR_REST_URL/query \
  -H "Authorization: Bearer $UPSTASH_VECTOR_REST_TOKEN" \
  -d '{
   "vector":[0.9215,0.3897],
   "topK" : 5,
   "includeMetadata": true,
   "filter": "url GLOB \"*imgur.com*\""
}'

See Metadata Filtering documentation for more details.

Range Example

Python
JavaScript
Go
PHP
curl

from upstash_vector import Index

index = Index(
    url="UPSTASH_VECTOR_REST_URL",
    token="UPSTASH_VECTOR_REST_TOKEN",
)

index.range(
    cursor="0",
    limit=3,
    include_metadata=True,
)

import { Index } from "@upstash/vector"

const index = new Index({
  url: "UPSTASH_VECTOR_REST_URL",
  token: "UPSTASH_VECTOR_REST_TOKEN",
})

await index.range({
  cursor: "0",
  limit: 3,
  includeMetadata: true,
})

package main

import (
	"github.com/upstash/vector-go"
)

func main() {
	index := vector.NewIndex("UPSTASH_VECTOR_REST_URL", "UPSTASH_VECTOR_REST_TOKEN")

	index.Range(vector.Range{
		Cursor:          "0",
		Limit:           3,
		IncludeMetadata: true,
	})
}

use Upstash\Vector\Index;
use Upstash\Vector\VectorRange;

$index = new Index(
  url: 'UPSTASH_VECTOR_REST_URL',
  token: 'UPSTASH_VECTOR_REST_TOKEN',
);

$index->range(new VectorRange(
  limit: 3,
  includeMetadata: true,
));

curl $UPSTASH_VECTOR_REST_URL/range \
  -H "Authorization: Bearer $UPSTASH_VECTOR_REST_TOKEN" \
  -d '{ "cursor" : "0",  "limit" : 3, "includeMetadata": true}'

{
  "result": {
    "nextCursor": "4",
    "vectors": [
      { "id": "id-0", "metadata": { "url": "https://imgur.com/z9AVZLb" } },
      { "id": "id-1", "metadata": { "url": "https://imgur.com/a2nCEIt" } },
      { "id": "id-2", "metadata": { "url": "https://imgur.com/zfOPmnI" } }
    ]
  }
}

Data

Data is another kind of information you can store per vector to attribute some context to it. Compared to metadata, it is not structured, and it can only be fetched in queries, not used to further filter them. It is especially useful when you upsert raw text data, so that you would have access to the textual form of vector along with the embedded vector values. It can save you from storing contextual information per vector in a separate database. You can set both the metadata and data, or only one of them while upserting your vectors as follows:

Python
JavaScript
PHP
curl

from upstash_vector import Index

index = Index(
    url="UPSTASH_VECTOR_REST_URL",
    token="UPSTASH_VECTOR_REST_TOKEN",
)

index.upsert(
    [
        {
            "id": "id-0",
            "vector": [0.9215, 0.3897],
            "metadata": {"url": "https://imgur.com/z9AVZLb"},
            "data": "data-0",
        },
        {
            "id": "id-1",
            "vector": [0.3897, 0.9215],
            "data": "data-1",
        },
    ],
)

import { Index } from "@upstash/vector"

const index = new Index({
  url: "UPSTASH_VECTOR_REST_URL",
  token: "UPSTASH_VECTOR_REST_TOKEN",
})

await index.upsert([
  {
    id: "id-0",
    vector: [0.9215, 0.3897],
    metadata: {"url": "https://imgur.com/z9AVZLb"},
    data: "data-0",
  },
  {
    id: "id-1",
    vector: [0.3897, 0.9215],
    data: "data-1",
  },
])

use Upstash\Vector\Index;
use Upstash\Vector\VectorUpsert;

$index = new Index(
  url: 'UPSTASH_VECTOR_REST_URL',
  token: 'UPSTASH_VECTOR_REST_TOKEN',
);

$index->upsertMany([
  new VectorUpsert(
    id: 'id-0',
    vector: [0.9215, 0.3897],
    data: 'data-0',
  ),
  new VectorUpsert(
    id: 'id-1',
    vector: [0.3897, 0.9215],
    data: 'data-1',
  ),
]);

curl $UPSTASH_VECTOR_REST_URL/upsert \
  -X POST \
  -H "Authorization: Bearer $UPSTASH_VECTOR_REST_TOKEN" \
  -d '[
        {
            "id": "id-0",
            "vector": [0.9215, 0.3897],
            "metadata": {"url": "https://imgur.com/z9AVZLb"},
            "data": "data-0"
        },
        {
            "id": "id-1",
            "vector": [0.3897, 0.9215],
            "data": "data-1"
        }
    ]'

When a raw text data is upserted, the data will be set to the raw text data automatically:

Python
JavaScript
PHP
curl

from upstash_vector import Index

index = Index(
    url="UPSTASH_VECTOR_REST_URL",
    token="UPSTASH_VECTOR_REST_TOKEN",
)

index.upsert(
    [
        {
            "id": "id-2",
            "data": "Upstash is a serverless data platform.",
        },
    ],
)

import { Index } from "@upstash/vector"

const index = new Index({
  url: "UPSTASH_VECTOR_REST_URL",
  token: "UPSTASH_VECTOR_REST_TOKEN",
})

await index.upsert([
  {
    id: "id-2",
    data: "Upstash is a serverless data platform.",
  }
])

use Upstash\Vector\Index;
use Upstash\Vector\DataUpsert;

$index = new Index(
  url: 'UPSTASH_VECTOR_REST_URL',
  token: 'UPSTASH_VECTOR_REST_TOKEN',
);

$index->upsertData(new DataUpsert(
  id: 'id-0',
  data: 'Upstash is a serverless data platform.',
));

curl $UPSTASH_VECTOR_REST_URL/upsert-data \
  -X POST \
  -H "Authorization: Bearer $UPSTASH_VECTOR_REST_TOKEN" \
  -d '{
        "id": "id-0",
        "data": "Upstash is a serverless data platform."
      }'

Python
JavaScript
PHP
curl

from upstash_vector import Index

index = Index(
    url="UPSTASH_VECTOR_REST_URL",
    token="UPSTASH_VECTOR_REST_TOKEN",
)

result = index.query(
    data="What is Upstash?",
    include_data=True,
)

for res in result:
    print(f"{res.id}: {res.data}")

import { Index } from "@upstash/vector"

const index = new Index({
  url: "UPSTASH_VECTOR_REST_URL",
  token: "UPSTASH_VECTOR_REST_TOKEN",
})

const result = await index.query({
  data: "What is Upstash?",
  includeData: true,
  topK: 3
})

for (const vector of result) {
  console.log(`${vector.id}: ${vector.data}`)
}

use Upstash\Vector\Index;
use Upstash\Vector\DataQuery;

$index = new Index(
  url: 'UPSTASH_VECTOR_REST_URL',
  token: 'UPSTASH_VECTOR_REST_TOKEN',
);

$results = $index->queryData(new DataQuery(
  data: 'Upstash is a serverless data platform.',
  topK: 3
  includeData: true,
));

foreach ($results as $result) {
  print_r($result->toArray());
}

curl $UPSTASH_VECTOR_REST_URL/query-data \
  -H "Authorization: Bearer $UPSTASH_VECTOR_REST_TOKEN" \
  -d '{
        "data": "What is Upstash?",
        "includeData": true,
      }'

Similar to metadata, the data field can be requested in queries, range iterator, and fetch requests, by setting the includeData to true as shown above.

Vector

Documentation Index

​Metadata

​Data

Metadata

Data