Paginating a rate-limited legacy API into pgvector. Stream.resource is one of Elixir's most underrated tools:
Stream.resource(
fn -> 1 end,
fn page ->
case Client.list(page: page) do
{:ok, []} -> {:halt, page}
{:ok, items} -> {items, page + 1}
end
end,
fn _ -> :ok end
)
|> Stream.chunk_every(100)
|> Stream.each(&index_batch/1)
|> Stream.run()
Lazy, backpressure-friendly, composable.
