```
├── .genignore
├── .gitignore
├── ADR.md (2.2k tokens)
├── ATTRIBUTION.md (300 tokens)
├── BACKLOG.md (2.1k tokens)
├── BLOG.md (1700 tokens)
├── CHANGELOG.md (8.6k tokens)
├── CONTRIBUTING.md (400 tokens)
├── GETTING-STARTED.md (300 tokens)
├── GROWTH.md (400 tokens)
├── LICENSE.md (200 tokens)
├── README.md (2.1k tokens)
├── TODO.md (1800 tokens)
├── context.json
├── convention.md (400 tokens)
├── process-formdata.drawio.png
├── routers/
├── ADR.md (300 tokens)
├── COVERAGE.md (1500 tokens)
├── README.md (700 tokens)
├── StandardURL.ts (100 tokens)
├── github.ts (2.8k tokens)
├── npmjs.ts (800 tokens)
├── router-simple.drawio.png
├── router.drawio.png
├── x.ts (1300 tokens)
├── ycombinator.news.ts (1400 tokens)
├── uithub.ingestjson/
├── .gitignore
├── README.md (200 tokens)
├── globals.d.ts
├── main.ts (4.8k tokens)
├── package.json
├── public/
├── ext-to-mime.json (omitted)
├── test.js (700 tokens)
├── types.ts (100 tokens)
├── wrangler.toml
├── uithub.ingestsql/
├── .dev.vars.example
├── .gitignore
├── CHANGELOG.md (700 tokens)
├── README.md (400 tokens)
├── main.ts (4.9k tokens)
├── package.json
├── public/
├── default-genignore.txt (300 tokens)
├── ext-to-mime.json (omitted)
├── index.html (omitted)
├── openapi.json (2.1k tokens)
├── types.ts (200 tokens)
├── wrangler.toml
├── uithub.ingesttar/
├── .gitignore
├── README.md
├── genignore.ts (600 tokens)
├── globals.d.ts
├── main.ts (4.2k tokens)
├── package.json
├── public/
├── default-genignore.txt (300 tokens)
├── ext-to-mime.json (omitted)
├── index.html (omitted)
├── openapi.json (1400 tokens)
├── tarReader.ts (2.3k tokens)
├── test.js (600 tokens)
├── types.ts (100 tokens)
├── wrangler.toml
├── uithub.ingestzip/
├── .gitignore
├── ADR.md (700 tokens)
├── README.md (1000 tokens)
├── curlspeed.sh (100 tokens)
├── genignore.ts (2.1k tokens)
├── globals.d.ts
├── main.ts (5.5k tokens)
├── package.json
├── public/
├── default-genignore.txt (300 tokens)
├── ext-to-mime.json (omitted)
├── index.html (omitted)
├── openapi.json (1400 tokens)
├── test.js (500 tokens)
├── types.ts (100 tokens)
├── wrangler.toml
├── xymake.json (100 tokens)
├── uithub.merge/
├── .gitignore
├── README.md (500 tokens)
├── ingestlist.drawio.svg (omitted)
├── ingestlist.md (400 tokens)
├── main.ts (2k tokens)
├── package.json
├── wrangler.toml
├── uithub.otp/
├── README.md (100 tokens)
├── uithub.outputjson/
├── .dev.vars.example
├── .gitignore
├── README.md (100 tokens)
├── main.ts (1800 tokens)
├── package.json
├── test.js (1100 tokens)
├── wrangler.toml
├── uithub.outputmd/
├── .gitignore
├── README.md (500 tokens)
├── main.ts (1800 tokens)
├── package.json
├── test.js (1100 tokens)
├── wrangler.toml
├── uithub.outputzip/
├── .gitignore
├── README.md (300 tokens)
├── main.ts (2.9k tokens)
├── package.json
├── public/
├── openapi.html (omitted)
├── openapi.json (1600 tokens)
├── wrangler.toml
├── uithub.search/
├── .gitignore
├── CHANGELOG.md (100 tokens)
├── README.md (700 tokens)
├── main.ts (1800 tokens)
├── package.json
├── public/
├── openapi.html (omitted)
├── openapi.json (1600 tokens)
├── wrangler.toml (100 tokens)
├── uithub/
├── .dev.vars.example (100 tokens)
├── .gitignore
├── README.md (100 tokens)
├── buildTree.ts (700 tokens)
├── dev-cycle.drawio.png
├── github-to-md-pipe.drawio.png
├── globals.d.ts
├── homepage.ts (1100 tokens)
├── main.ts (4.7k tokens)
├── package.json (100 tokens)
├── public/
├── index.html (omitted)
├── ratelimiter.ts (700 tokens)
├── static/
├── 404.html (omitted)
├── 404.html.md (200 tokens)
├── 429.html (omitted)
├── 429.html.md (200 tokens)
├── aiswe.html (omitted)
├── android-chrome-192x192.png
├── android-chrome-512x512.png
├── apple-touch-icon-precomposed.png
├── apple-touch-icon.png
├── buttons.js (500 tokens)
├── explore.js (3.8k tokens)
├── favicon-16x16.png
├── favicon-32x32.png
├── favicon.ico
├── filter.js (1100 tokens)
├── mobile-not-supported.html (omitted)
├── openapi.html (omitted)
├── openapi.json (3.7k tokens)
├── overview.drawio.svg (omitted)
├── pin.svg (omitted)
├── plugins.html (omitted)
├── plugins.js (3.2k tokens)
├── plugins.json (700 tokens)
├── plugins.md (200 tokens)
├── plugins.schema.json (700 tokens)
├── privacy.html (omitted)
├── privacy.md (800 tokens)
├── profile.js (2.6k tokens)
├── push-file-object.json
├── push-file-operation.json (500 tokens)
├── resolve.html (omitted)
├── resolve.html.md (100 tokens)
├── search.js (4.6k tokens)
├── site.webmanifest (100 tokens)
├── threads.html (omitted)
├── threads.html.md (200 tokens)
├── tos.html (omitted)
├── tos.md (900 tokens)
├── uit-left.png
├── uit-transparent.png
├── uit.png
├── uithub2.mov
├── urlux.js (3.2k tokens)
├── view.html (omitted)
├── view2.html (omitted)
├── vscode.html (omitted)
├── tsconfig.json
├── types.d.ts (100 tokens)
├── uithub-flow.drawio.svg (omitted)
├── wrangler.toml (200 tokens)
├── xymake.json (100 tokens)
```
## /.genignore
```genignore path="/.genignore"
package-lock.json
uithub.ziptree/old
*.html
**/ext-to-mime.json
*.svg
```
## /.gitignore
```gitignore path="/.gitignore"
.git
node_modules
.wrangler
package-lock.json
```
## /ADR.md
## ADR / Decisions
### 2025-04-01 - Supporting binary data?
My initial decision when I started to build `uit` was to use JSON Sequence as the core format.
Support for processing binary data would at first be done using references over URL (raw.githubusercontent.com incase of github). Later we can add support for streaming binary data by indicating the length of binary in a json sequence item, after which the raw binary data will be streamed. See this [claude convo](https://claude.ai/share/b162b3c7-8996-4d08-9b38-e2af2e5e5e6c)
However, after an enlightening conversation with the great https://x.com/zplesiv, I decided to t may be better to use Multipart FormData https://claude.ai/chat/1c647deb-a65a-49f7-89fc-b1092e375328 https://claude.ai/share/8bbab353-d07b-4d50-a173-02084d0b2fbb !! This can also be streamed! https://github.com/ssttevee/js-multipart-parser/tree/master !! The advantage is that binary data will be better supported out of the box. Also, this is kind of markdown already, which is quite cool.
- ✅ built simplified library based off of https://github.com/ssttevee/js-multipart-parser for most easy use, anywhere.
- ✅ Create a guide specifically for Cloudflare Workers.
- ✅ redo uithub.ingestzip using this guide, see if it actually streams as `multipart/formdata`.
- ✅ redo uithub.search to output MultiPart formdata.
# 2024-11 - name 'uithub'
uithub is the client for the UIT. it started as a simple 'g to u' trick with the URL, but UIT has grown past github alone, as it now also supports other domains via the UIT convention. Uithub is therefore an even more logical name than before.
Previously I was worried about GitHub taking a negative legal stance against uithub; here's the gist:
https://claude.ai/chat/349a5d27-4978-47fe-873c-bacd24ebec11
I'll keep uithub.com until they take legal action....
Claude:
I understand your strategic thinking here. Let me break down this approach pragmatically:
Potential Benefits:
- Easy discovery through URL similarity
- Built-in trust transfer
- User convenience
- Email list as backup
Mitigation Strategies:
1. Having a backup plan:
- Alternative domain ready (like codagent)
- Email list for user communication
- Clear communication strategy for potential transition
2. Risk management:
- Keep solid documentation of all users
- Have export/backup systems ready
- Maintain transparent communication
- Have transition plan ready
3. Business continuity:
- Email list as primary user asset
- Clear terms of service
- Documented service value beyond just the name
- Strong actual product offering
Practical Advice:
1. If you proceed:
- Be extremely clear about non-affiliation
- Document everything meticulously
- Keep legal defense fund ready
- Have technical migration path ready
- Keep backup domains registered
2. Risk minimization:
- Don't use GitHub logos/branding
- Keep all marketing materials distinct
- Document user awareness of non-affiliation
- Keep operations clean and professional
This is still legally risky, but I understand your business reasoning. Just ensure you're prepared for potential consequences and have solid contingency plans.
Would you like me to elaborate on any of these aspects of the strategy?
# 2024-11 - Pricepoint (Old)
Is pricing for regular requests not too expensive? Maybe should make it 10x cheaper: 0.1 cent per request. This is more similar to e.g. scrapingbee. However, let's see first with this pricing (5000x that of upstash)
The reason it's good is:
- we're giving the first $10/month at a 10x cheaper rate
- we need room to give massive discounts to enterprise; b2c doesn't need to be cheap at scale, as they won't have scale
- 1 cent per additional request is fair, won't cost a dollar for an entire day of regular use. and you won't normally get to this much traffic unless you're really building something bigger
- $10/month now gives 10k requests which is 333 per day on average, which should be more than sufficient.
# 2025-04 - Monaco?
Maybe looks way cooler than raw text! However, may also be much harder to make that stream in... Let's see.
Seems possible to stream to monaco through adding every chunk: https://claude.ai/share/ee1cd1f2-b94c-4340-9c0e-85d0d05d0a7c
This'd make things really pretty and readable!
Edit: tried it, but slows things down as well, and in-md codeblocks didn't highlight easily
# 2025-04-21 - Error handling
Error handling sucks. how to improve?
- ❌ Files that error out should maybe add `/.errors/extension-name/...path` files so errors can instantly be shown
- ❌ We could also conventionalize concatenating all errors and adding them as a last file. This would be
- ❌ Trailers (headers at the end) are another option so it doesn't become a file. However, this could be trickier working with.
- ✅ `x-error` with format `{module-id};{status};{message}` should be passed on and if encountered, shouldn't be filtered or processed, so we can see errors for every individual file, where they happened, and with what file input. Perfect! 😃
# 2025-04-21 - URL Chain Auth Pass Sucks & Need standardized way to charge
I now have a chain of urls that I pipe the request through, a "urlpipe". The problem now is auth should be passed all the way down the chain for this to work. This means I MUST own the hosting for this to be trustworthy. It would be much better if I could stream in the zip output into the last one and stream it back, just like that. Maybe this can be done by adding a temporary token to retrieve back the auth token on the other end, but that's also a lot of extra complexity.
I don't know yet.
Doing body-body pipe in a single worker may slow down things but haven't tested yet. In the end it may be better as data comes from any source. Try if I can get it to work, and benchmark on speed. AI should be able to get it done and generalize from an array of requests that should flow into each other with custom headers and query parameters.
If that does not work out, brainstorm to make URL chain more secure and scalable, less error-prone.
In devmode, it'd be very cool to be able to see the intermediate result for any request as an explorable hierarchy. We could do this by creating clones of the responses and streaming those into a zip which can be made viewable in the browser. However, this could be done with some sort of devtool, and does not need to be part of the spec.
URL Chain sucks because:
- It mixes query params. Can be seen as a regression OR a feature. Not a real problem yet, but could become one for larger pipes with third-party creators.
- It requires me to pass on the authorization header. The scope of the authorization should be as minimal as possible for each processor so this isn't ideal.
**Need standardized way to charge**
We cannot use x-price as response header as most servers would not know the exact price in the beginning. Besides that, there's no good way to track usage by which part of the chain right now.
Possible ways to solve it;
- ❌ Trailer (header at the end) that specifies total cost incurred **probably worst option since it'd have a single price at the end and connection could be closed early**
- ❌ Optional FormData header for `x-price` that accumulates over time so we know cost intermediate as well. When received formdata already contains this it shall just be ignored and overwritten.
- ✅ Server self-manages: Send along `x-monetary-url` header to server that it can use with sponsorflare to add balance from to themselves and deduct it from the user. Along with expected max cost based on size and openapi spec, this can be a great way, since it allows a lot of freedom on how to charge, while respecting privacy of the user.
✅ Proposed solution:
- At the start, create an OTP and send that along as x-source-authorization header
- Create a max-budget for the entire request and send that along as x-monetary-url header
- every module then uses the monetary url with the uit-murl service to withdraw the required amount
- the module that needs direct access to the source (ingestzip, for example) would use the x-source-authorization together with the otpproxy to retrieve the zip (can be done just once)
🔥 This looks very promising and I'll try to implement it this way using `uithub.murl` and `uithub.otp`.
# 2025-04-21 - No easy getting started / docs
How to run uithub and improve individual components locally? How to easily develop a new plugin?
- Try right now to clone uit and make it easy to run locally. A fallback on the individual services would be great, not requiring running them per se.
- Figure out if I can do a check whether or not service-bindings are connected. If possible, make hostname a configurable setting, and make service binding connection optional falling back to regular fetch.
- Ideal situation: you only need to run 1 service (uithub) to run locally, and switch to any additional service to use it as part of the pipeline.
# github master zip redirect
Master redirects to production branch. does master always redirect to the deafult branch?????? for zip, maybe https://github.com/cloudflare/cloudflare-docs/archive/refs/heads/production.zip but for other things, definitely not.
# 2025-04 - Pricing / ratelimiting
Based on what should plugins be free:
- repos over 1000 stars
Based on what should features require pay-as-you-go
- private repos
- organisation repos
- repos under 1000 stars
The premium features should require deposit of $50
Also add button to unlock for entire organisation/owner (contact me)
# UIT vs. ActionSchema
Actionschema was json-based and because of it, it got the side effects of state at arbitrary public urls. this protocol potentially removes this problem using the formdata standard, while also making things stream through more easily, by design. They're two different beasts that solve the problem in different ways.
Let's lay them against each other and figure out how they measure up and how we can get the best of both worlds, in one.
# Ability to get access to organisations.
Tried repo scope access with a new oauth client:
https://github.com/login/oauth/authorize?client_id=Ov23lipC8pwTAzCrbOGH&scope=repo&redirect_uri=https://janwilmake.com/callback
But for shapleyai, I get:
`Until access is granted by an owner, the application cannot access the organization’s private data or modify its public data.`
When I click "Request", it says:
```
Send request for approval?
Owners will receive an email regarding your request.
By requesting this approval, you are asking owners of the shapleyai organization to allow this application to access private organization data and modify public organization data.
```
This means many organisations have this problem. Users should first request it and be allowed.
## /ATTRIBUTION.md
# Attribution Guidelines
While the Universal Information Terminal (UIT) is released under the MIT license which only requires preservation of copyright notices, we kindly ask that you provide attribution when using or building upon this project. This is not a legal requirement, but a courtesy that helps our project grow and encourages continued development.
## Preferred Attribution Methods
### In Code Projects
When incorporating UIT or its modules in your codebase, please include a comment in your main files:
```
This project uses Universal Information Terminal (UIT) by Jan Wilmake
https://github.com/janwilmake/uit
```
### In Documentation
When referencing the project in documentation, READMEs, or other written materials:
> Built with [Universal Information Terminal (UIT)](https://github.com/janwilmake/uit) by Jan Wilmake
### In Applications
For applications with a user interface that incorporate UIT functionality, consider adding to your "About" page or settings section:
> Powered by Universal Information Terminal (UIT)
> https://github.com/janwilmake/uit
## Social Media
When sharing projects that use UIT on social media, we appreciate a mention or tag:
- X/Twitter: @janwilmake
## Why Attribution Matters
Attribution helps:
- Grow our community of users and contributors
- Provide recognition to the developers who built the tools
- Help others discover useful open source software
- Encourage continued maintenance and improvement
Thank you for respecting these guidelines. If you have questions or need clarification about attribution, please open an issue on GitHub.
## /BACKLOG.md
# Improving the markdown output
- Add `maxTokens` filter to `ingestzip`, but ensure it still browses through the pathnames (but skipping content). This is more efficient than doing it later on and will ensure we get the full tree still. **🤔 Not sure! maybe its better to do this after we built up the tree. For formats other than markdown, maxTokens isn't that important. Maybe the tree should also be something that we always append/prepend as a file. Pretty useful**
- Add ability to omit filtered files out of the tree when it makes sense (see https://x.com/janwilmake/status/1916841093162831924). Goal would be to get the tree that makes most sense under 10k tokens for any repo, even for bun.
# Creating sha-cached basePath-based inferences (big on performance)
Figure out how I can nicely benchmark speeds in different ways of the different plugins and the time for estabilishing the connection between the different workers. Do another deepdive on how to make this as fast as the original uithub on vercel.
❗️ Found that `initialResponseTime` for `ingestzip` service is often in the ballpark of: 1750-2250ms. Caching the zip into KV or R2 makes a ton of sense to reduce this to 100s of ms.
1. the url + query is the unique identifier
2. we can destructure the url and make it incrementally more generic, the source being the most generic
3. when something needs generating, we can find the closest generation first, then generate from there to save on compute
4. when someone visits a repo for which the latest sha hasn't been indexed yet, all plugins and obvious paths should be added to a queue to pre-generate them. This makes everything superfast.
The logic of what to generate should be surfaced to give organisations or owners control. Pregenerating is an upfront investment that has future benefit to others, and could be seen as a business in itself. What would you pregenerate for others?
Only do the above for public repos with permissive lisence allowing commercial redistribution (for now).
> [!WARNING]
> Caching introduces legal risk
# X Plugin
1. give context of x threads found in the data
2. determine the key keyword or keywords that identify this repo
3. Find the X account(s) linked to the owner (and if that's an organisation, at least the core contributor, but if not, also look at top 50% contributors or so).
4. Use keywords within context of posts of X accounts to filter out threads that are relevant (without duplication).
5. Run a search on x with results of people mentioning the keyword(s) to find other mentions about this repo.
All of this should be done respecting privacy and with an xymake configuration in the repo. This will be a challenge, but very cool nonetheless!
# Ratelimiter is slow
Keep watching ratelimiter speed. Sorta slow sometimes, maybe ratelimt resets too fast which causes required reinitialisation which takes a bit longer?
# Process `.gitattributes`
After some research I found that https://github.com/public-apis/public-apis doesn't give the README file (280kb) because of their .gitattributes setting
If present we need to process it (see https://claude.ai/chat/1ad5ee29-7ea4-4dce-a61f-02a2aa582189)
1. point to the raw githubusercontent url for files which are ignored from the ZIP
2. If any LFS pointer files are present, this will hint us there are other bigger files. Use https://uithub.com/{owner}/{repo}/raw/{branch}/{...path}
And make the raw endpoint: https://uithub.com/{owner}/{repo}/raw/{branch}/{...path}
```ts
const apiUrl = `https://api.github.com/repos/${owner}/${repo}/contents/${path}`;
const apiResponse = await fetch(apiUrl, {
headers: {
Accept: "application/vnd.github.raw",
},
});
```
Confirm that it works without api key... Also allow passing an API key header/cookie/queryparameter
https://docs.github.com/en/repositories/working-with-files/managing-large-files/about-git-large-file-storage
# Fast File Retrieval
A core problem that is also core to uithub is fast file retrieval. How I solve it now is by always getting the latest zip directly from GitHub. This is slow for big zipfiles. At a minimum, the latency is transfering the zip to my location which can only go with a 100mb/s or so I think.
Building a service on top of GitHub is hard because it's hard to know whether or not their zip files are updated. For my own repos I can use a webhook to refresh stuff, for public repos I can use github events. **The webhooks take maximum several seconds**, p95 is under a second. It's not instant. The events for public repos are delayed [at least 5 minutes](https://github.blog/changelog/2018-08-01-new-delay-public-events-api/) and may be [up to 6 hours](https://github.blog/changelog/2018-08-01-new-delay-public-events-api/).
When someone pushes something to GitHub, that is the source of truth. If that person is authenticated with my services, I can be sure to have the newest version within a few seconds, as I can immediately propegate it to my own services. However, for public events, that is not possible, and I cannot guarantee things to be up-to-date if I don't use the original zip.
For the purpose of uithub, it's quite feasible still to use the zip for most smaller repos, but for larger ones, it's kind of annoying, as it can take dozens of seconds with ease.
If we want to cache things, we have multiple options. The question is how long we'd want to cache because it would take a large amount of storage. For something like uithub and also for viewing a website quickly, I think maybe redis is great.
Pricing for Upstash Redis is $0.25 per GB-month. If we would just store something for an hour each time, and we do that 100 times a day with repos 1GB each, 100GB-hour, 0.13GB-month, so 3.4 cent per day, €1 per month. That's a lot. But what if we just store it for 10 minutes? Or just 1 extra each time you reload the page? Reduced cost a lot, small reduction in usability. This seems interesting. I'm basically buying myself a working memory for the edge. $0.000005 or 172k GB-minutes for $1. If 1 user needs 1GB, that's basically 172k user-minutes for $1. Nice to calculate it like that. 29 user-hours for 1 cent. If you look at it like that, and we can actually use the redis in this way, it's damn cheap.
So how do we actually get it like that?
- Max commands per second: 1000
- Max request size: 1mb
- Max record size: 100mb
- Max data size: 10gb
- Oh wait.... max 200GB bandwidth after which i pay 0.03-0.10 per gb. this is the bottleneck that makes it incredibly expensive.
Ok... So Cloudflare KV is also expensive... It's 0.50 per GB-month. But R2 is $0.015 per GB month! But R2 takes 30 seconds to propagate I read somewhere. Is this true?
ARGGHHHHHH all these different factors make this a very complex problem! In the end there are just so many variables... The implementation also depends a lot on the usecase... Maybe I should read about it a bit about other people's experiences. I wish there was just a way that you could just write down your usecase and the AI would automatically find the best implementation with all this experimentation, etc. We're almost there!...
# Making zip subset retrieval faster
[About zip archives](https://docs.github.com/en/repositories/working-with-files/using-files/downloading-source-code-archives#stability-of-source-code-archives). Maybe I can do this:
- If I place the retrieval where the zip is.
- If I have the earliest possible filter
- If I use another language to do this such as rust
My original source is Github. The zip comes from there which is usually pretty fast.
After some tests, I found that retrieving a file from a zip can be incredibly fast. It takes less than a second to get https://uithub.com/file/facebook/react/blob/main/README.md and 2 seconds to get https://uithub.com/file/facebook/react/blob/main/compiler/packages/make-read-only-util/src/__tests__/makeReadOnly-test.ts even though the zipfile is an insane 600MB or so, compressed.
Doing some more tests from iad1 (us-east) I found that retrieving the zip via a fetch takes 50ms for small zips (claudeflair) versus 300ms for large ones (oven-sh/bun). However, parsing through the entire zip takes an additional 10ms for small zips (bun) versus 10 seconds for large ones (oven-sh/bun). After retrying I can see the zip of bun only takes 80ms to retrieve (may be cached by vercel or github), while the parsing of the zip still takes 8.7s for the image. However, if we encounter the file earlier, we now return early, which is a great advantage for things like `README.md`. This is all in production on max 3GB ram.
# Improve GitHub URL to Zip mapping
Issue: not all branches are accessible yet on github and this is actually quite some logic! Many will actually give a dead link, which is problematic! Since we have more than zipobject alone for github zip finding, this should become a service or lib in itself. Maybe, its better not to allow github URL in the first place, or at least, we should be very clear on what is supported from the github URL structure and what isn't.
Possible github URLs in browser:
- https://github.com/facebook/react
- https://github.com/facebook/react/tree|blob/main/[path]
- https://github.com/facebook/react/wiki/[page]
- https://github.com/facebook/react/tree/18eaf51bd51fed8dfed661d64c306759101d0bfd
- https://github.com/facebook/react/tree/gh/mvitousek/5/orig/compiler (branch can have strange characters including `/`)
- https://github.com/facebook/react/tree/v16.3.1 (it's a tag)
Two strategies are possible to figure out the zip url and raw url:
1. trial and error; try most likely and try other possibilities later to finally get the zip url. the tricky part is that https://codeload.github.com/facebook/react/zip/refs/ANYTHING will always redirect even if it didn't exist, so we need to follow the redirect.
2. use `git.listServerRefs`. If we cache it and But this easily takes half a second...
It's best to create a function to do this trial and error. This would most likely just be ratelimited by 5000 req/hour/ip. Additionally we could cache the tagnames and branchnames - but not the shas they're tied to. However, I don't think this is worth the additional complexity as the amount of trials before a hit is likely between 2-3 on average (assuming we start with 2 in parallel).
# UIT cli
Installs context from url into standard location `.rules`
# version selectors
- a github repo has branches and versions
- a npm package has versions
## /BLOG.md
Inspo: https://www.gitlip.com/blog
# Formdata Blogpost (draft)
Draft outline:
- short intro
- how uit has evolved over time, different things i tried
- why i think formdata is the best way to do multi-file streaming/processing
- whats formdata and what its capable of
- formdata history
- intro to uit framework and uuithub(uber uithub, advanced version, to be renamed at later stage)
- attribution
# Zoran
That's a good list! Some of my thoughts:
- streaming is particularly suitable for serverless environments where resources are low (can process files bigger than the memory of the worker)
- might be worthwhile to take a look at popular APIs and suggest that some of them (e.g. OpenAI image / audio APIs) should use multipart/form-data instead of their base64 approach which is wasteful in terms of memory and CPU and is not suitable for streaming
- (you're likely aware, but to mention) multipart/form-data is in the multipart family of Content-Types. It's strange to me that multipart/form-data is the recommended go-to type for general purpose use - I would've expected multipart/mixed to be that...
- I think streaming parsing is particularly interesting. Async generators in Javascript...
- worth mentioning that support for streaming form-data in browsers (and related runtimes) is not present
- If I was writing it, I might have gone as far as to claim that multipart/form-data should be the standard format to feed LLMs (don't know what your thoughts are about this). When I write prompts to LLMs, I most often default to something like ### START: file-1 ### and ### END: file-1 ### to delimit files. multipart/form-data is exactly that but standardized and the borders can be adjusted as necessary
# Claude
This is a great start for your blogpost outline! To make it a potential top Hacker News story, you'll want to balance technical depth with engaging storytelling, while adding some unique insights that spark discussion. Here are some suggestions to enhance your outline:
## Enhanced Outline for UIT/FormData Blog Post
### Introduction
- Open with a compelling problem statement about file processing challenges in modern cloud environments
- Include a brief "aha moment" that led you to explore FormData as a solution
- Tease the unique perspective or insight readers will gain
### Evolution Journey
- Share your personal journey with UIT - what specific pain points led to its creation
- Highlight the technical iterations and dead ends (zipobject.vercel, shadowfs, filetransformers)
- Frame it as a technical detective story - the search for the right abstraction
### The FormData "Revelation"
- Expand on why FormData emerged as the superior solution
- Contrast with alternatives you tried (JSON-based approaches, other streaming methods)
- Include performance benchmarks or real-world examples showing the difference
### FormData Deep Dive
- Explore FormData's origins and how it was never intended for this use case
- Explain the "hidden power" of FormData headers that most developers overlook
- Include code snippets showing FormData in traditional usage vs. your innovative approach
### The UIT Framework
- Present UIT as a mental model first, then a technical implementation
- Emphasize the 4-step process and why modularity matters for serverless architectures
- Include a visual representation of how modules can be composed
### Real-World Applications & Performance
- Show concrete examples of UIT solving previously difficult problems
- Include performance metrics (memory usage, processing times for large files)
- Discuss specific edge cases where the approach shines
### Future Directions
- Share upcoming modules or extensions to the framework
- Discuss potential integrations with other systems or standards
- Present open questions or areas where you still see room for improvement
### Community & Open Source Strategy
- Explain your vision for community contribution
- Highlight where you see the most need for collaboration
- Discuss your approach to documentation and making the project accessible
### Conclusion
- Circle back to the initial problem and how UIT represents a step forward
- End with a thought-provoking question or call to action for readers
### Technical Elements to Include:
- Diagrams showing data flow through the system
- Code snippets demonstrating key concepts
- Benchmarks comparing UIT to other approaches
- "Gotchas" and lessons learned during development
For a Hacker News audience, focus on the novel technical insights, unexpected discoveries about FormData capabilities, and architectural decisions that led to your current approach. The HN community especially appreciates posts that teach them something new about a familiar technology or challenge conventional wisdom with solid technical reasoning.
Would you like me to elaborate on any specific section of this enhanced outline?
# X Thread (leading to blogpost and repo)
# Get back to users after launch
https://x.com/janwilmake/status/1895375026939142292
Make a pinned post like this one: https://x.com/samwillis/status/1824385175486337137 a thread that introduces me to my new followers and explains my mission. also, hackernews and producthunt.
# X Thread
(🧵 1/8)
Excited to announce that UIthub is now open source! 🎉
We're building the future of code analysis and development, starting with the entire GitHub ecosystem.
(2/8)
What is UIthub? Think GitHub, but with a U - we're creating a powerful platform for extracting, analyzing, and interacting with code from any repository.
(3/8)
Our mission: Enable a community of builders to create the next generation of code analysis tools. We believe in augmenting developers with AI while ensuring quality and reliability.
(4/8)
🔍 Key Features:
• Smart code extraction from any repo
• Direct code chat interface
• Rich context for LLMs
• Comprehensive analytics dashboard
(5/8)
The future of software development will be collaborative - between humans and AI agents. UIthub provides the foundation for building and validating these tools.
(6/8)
We're working on making it incredibly simple to write code with UIthub. New features coming soon will revolutionize how developers interact with codebases.
(7/8)
Join us in building the future of software development. Check out our repo, star us on GitHub, and start contributing today!
[GitHub Link]
(8/8)
Launch also live on:
• Product Hunt: [Link]
• Hacker News: [Link]
Let's build something amazing together! 🚀
# For Hacker News:
Title: UIthub – Open-source platform for code analysis and AI-assisted development
Show HN: I've been building UIthub, a platform for advanced code analysis and AI-assisted development, and today I'm making it open source. UIthub makes it easy to extract and analyze code from any GitHub repository, providing rich context for LLMs and enabling direct code chat capabilities.
The goal is to create an ecosystem where developers can build powerful code analysis tools on top of GitHub's dataset. This will be crucial for developing and validating software development agents in the future.
Key features:
- Smart code extraction from any repository
- Direct code chat interface
- Rich context generation for LLMs
- Analytics dashboard
- Coming soon: Simplified code writing experience
I believe the future of software development will be highly collaborative between humans and AI agents. UIthub aims to provide the foundation for building and validating these tools while ensuring quality and reliability.
Repository: [Link]
Documentation: [Link]
Would love to hear your thoughts and feedback!
# For Product Hunt:
Title: UIthub - Open-source code analysis platform
Tagline: Extract, analyze, and chat with code from any GitHub repository
Description:
UIthub is now open source! We're building a platform that makes it easy to analyze and interact with code from any GitHub repository. Our mission is to enable developers to create powerful code analysis tools and lay the groundwork for reliable AI-assisted development.
What makes UIthub special:
• Smart code extraction
• Built-in code chat interface
• Rich context for LLMs
• Comprehensive analytics
• Active community focus
We're committed to building an ecosystem where developers can create innovative tools for code analysis and AI-assisted development. Join us in shaping the future of software development!
[Include relevant screenshots and links]
## /CHANGELOG.md
# 19th June, 2024
- first version in https://github.com/janwilmake/github-contents-worker
- second version in https://github.com/janwilmake/github-contents-api
The first version, available at https://github.com/janwilmake/github-contents-worker was hosted at Cloudflare.
However, it became evident quickly that Cloudflares woker limits are too limited for downloading a large zipfile and unzipping it.
I therefore changed everything to use the [Vercel Functions](https://vercel.com/docs/functions). The first problem I encountered is it's not so straightforward to create a catch-all endpoint - something that was easy with Cloudflare, due to the routing mechanism Vercel and Next has implemented everywhere.
However, I found a neat little trick; my setting my `vercel.json` to this, ensures everything leads to `index.ts`.
```json
{
"$schema": "https://openapi.vercel.sh/vercel.json",
"rewrites": [
{
"source": "/:path*",
"destination": "/api/index.ts"
},
{
"source": "/",
"destination": "/api/index.ts"
}
],
"functions": { "api/*.ts": { "maxDuration": 60, "memory": 1024 } },
"public": true
}
```
Also, the max duration of the function is now 60s, the max for the free plan. This is quite a lot.
## Streaming instead of naive unzip library
After hosting it on vercel, things still didn't go well. After switching `fflate` into `unzipper`, things become much faster and I can now download big repos of over 100MB without timing out.
# 20th June 2024
I tried isomorphic-git. However, it doesn't support `--filter blob:none` yet which makes it too slow to put on a vercel function.
See https://github.com/isomorphic-git/isomorphic-git/issues/1123 and https://github.com/isomorphic-git/isomorphic-git/issues/685#issuecomment-455423505
Also, `git` through `child_process` also doesn't seem to work on vercel easily.
If that would work, maybe this would be a great way to get all logging information for any repo, including recent changes and all kinds of statistics.
For now let's let it pass.
# 24th of June, 2024
Discovered github has this feature too, at least for PR's:
> ProTip! Add .patch or .diff to the end of URLs for Git’s plaintext views.
When did this get added? Is it part of the API? Let's proxy this feature so I have it documented.
# 15th of August 2024
third version in https://github.com/janwilmake/github-registry
Today I've made a breakthrough! This is now not only a github contents api anymore, but a full fledged github registry!
- It allows registering a new user into the registry, after which search engines can pick this up.
- It allows finding OpenAPIs based on the repo homepage.
- It allows fetching all repos of a user
- It allows private and public repos
## Create GitHub Registry
- ✅ Add redis-kv and /register
- ✅ remove .env PAT
- ✅ Add /remove, /register, /registry, /[owner]/registry, /[owner]/repos
- ✅ Add Redis .env
- ✅ Create the OpenAPI for this
- ✅ Register myself via openapi.html
- ✅ Test other endpoints
🎉 I can now very easily navigate through my repos, by category and see the high level 🎉
## Contents
- ✅ Get profile using the apiKey. Find how to get profile on ActionSchema and in code
- ✅ Merge `github-contents-api` into this so that one can be archived! Ensure it works with public repos first of all
- ✅ Also make it work with private repos for registered users
- ✅ Archive old repo and put readme, ideas, and proper routing
## OpenAPIs
- ✅ Any repo is now an OpenAPI that can be used for agents.
- ✅ Any user now has an OpenAPI that can be used for agents.
- ✅ Added repo page
# In-repo search (needed for better chat-completion)
- ✅ filesize hierarchy
- ✅ Adhere to `.genignore` in same way `.gitignore` works
- ✅ Groq API
- ❌ Disallow search for repos that are too big (zipsize or filecount) **wait until i encounter this problem**
- ✅ Hierarchy pruning
- ✅ Update OpenAPI: ensure it's well-documented
# Make it usable & fun
✅ added repo.html and chat.html
✅ The repolist main link should be going here
# Small useful improvements website github (august 17th, 2024)
- ✅ searchbar
- ✅ preload
- ✅ Show "Github username" input with search icon and 'Let's talk' CTA
- ✅ Instead of asking for PAT immediately, hide this at footer, small
# September 8, 2024:
Moved away from this in favor of irfc.cloud
# September 25
Moved away from the github logic in irfc.cloud and made the simplest version into uithub.com: just unzip with some filters! The power lies in the fact that this is so easy to use and it's super scalable as it works on serverless and is fully stateless.
# October 9
✅ Incase Accept header includes text/html, call API and embed it in `view.html` where its printed; otherwise, route to API.
✅ Create `view.html` where the same url is used to fetch text/plain and shows that in a codeblock and a large copy button. Looks like a text-plain page but isn't quite as it'll have a subtle topbar.
✅ Show tokensize
✅ Helps you add more filters easily to trim down the context length (click in tree)
✅ Added maxTokens filter and quicklink to docs
✅ Before deploying this, be sure that the original API remains unchanged. People already use it as simple API.
✅ Header menu: In the header, add a menu for:
- format -> json, yaml, text (new tab)
- maxTokens -> text input
- maxFileSize -> text input
- ext -> filter on extension found in current output and press save
- location -> filter on tree (make it available in JS) and onclick, go there.
# November 4th, 2024
✅ refactor to allow for smaller files.
✅ send issues and comare to alternate response functions
✅ implement them one by one using ai
✅ deploy and test minimal things
# Issues, compare, push (november 5th)
✅ Set provided apiKey-query and header as a cookie. try getting apiKey from cookie too
✅ Test creating issue (also in other person repo)
✅ Test comment (also in other person issue in other person repo)
# Push (november 6th)
✅ Test push via test file. test overwrite, test new branch, test every use case. Ask AI to make it.
✅ Fix: A renamed file gets no content
✅ Extrahere file editing functionality and add proper testing... Test multiple edits in longer file, adding lines, removing lines, changing multiple lines in different amount of lines. Test this independently of git first to confirm it's functional.
✅ Creation of a new repo based on an external repo (in push: create repo, create initial commit, if needed. Ideally this is a fork)
✅ ensure to check if we have write permission to the owner/repo
✅ if not, ensure to fork it and create the new branch in the resulting owner/repo (that is our own)
✅ Define Push Context Better
✅ Test that forks https://github.com/brunabaudel/01 then creates a new README for it, then makes a PR. The crucial part is, if we can read but can't write, fork and PR needs to happen automatically. If we can write though, things can be written directly.
🎉 This is super useful. I can now make PRs to any public repo with ease.
# Login HTML screen if 401 (nov 7)
✅ Try submitting a cookie that stays good forever to a backend endpoint
✅ If it works, use that cookie as auth. we may now be able to see private repos
✅ See if this cookie stays even after closing browser and opening again, even after days
✅ In https://uithub.com/login.html show page to login/logout with details on cookie.
🎉 If this works I don't need localStorage and can keep doing stuff serverside. big achievement and useful everywehere!!!
✅ If we get a 401 and accept includes HTML (browser) let's show a little input box and link to get a PAT. Ensure to tell people it only gets stored as cookie and we don't store or process it on our server.
# LLM config screen if 402 (nov 7)
✅ If we don't have LLM config setup yet, show a 402 error. Regularly that should be JSON with some instructions on how to bypass, but in browsers it can be showing a HTML that allows to select some popular LLM basePath + model combos, and a link to easily create an API key.
🤔 This will be the place where I can later monetize by showing a 'recommended' setting. This setting would navigate the user to a page where they can fill in stripe payment details, and retrieve an API key for "ClaudeFlair", which basically always gets you the best model available with improved finetues based on the task. We can literally just use our own Anthropic API key in the back and charge the same, plus an extra charge or percentage for requests and LLM calls.
# Issue/discussion/pull details in JSON, YAML, Markdown or HTML (nov 7)
✅ Provide accept/maxTokens to all issue endpoints that need it (and compare, and message)
✅ Ensure the GET things respond with a JSON rather than a Response
✅ Ensure this works if called from a browser and allow similar accept query param that defaults to HTML in the browser that shows things as markdown, with the header.
# Issue/discussion/pull HTML (nov 7)
✅ Create a HTML that nicely shows the entire issue/discussion/pr but in an easy to copy markdown format.
👀 Allows very useful PR Creation Agent that's very easy to use.
Flow:
- Create or navigate to an issue or discussion
- Replace 'g' with 'u'
- View issue again and click 'resolve'
- (first time) login with github ==> will set apiKey
- (first time) purchase LLM usage ==> will set model, basePath, llmApiKey
- Show message.html
# Pulls is broken (november 9, 2024)
✅ Fetching pulls seems to return a 404. Debug.
✅ Deploy.
# Limit tokens by default (november 9, 2024)
- ✅ Default to 50.000 tokens max (good for most free models)
- ✅ show this number on site if no param is given. If you remove it, it should set it to 10000000.
- ✅ If the tokens is limited by default, ensure an extra warning is shown in the header
- ✅ If however JSON or YAML is requested, we'd want to default to no limit.
- ✅ Confirm now that regular HTML pages load faster (good for SEO and UX)
# uithub main API improvements (november 9, 2024)
✅ uithub `tokens` in json
✅ Add `?lines=true`: This query param should prepend each line with the line number in a way such that the LLM can easily see that. Needed for `message` endpoint
# X posts (november 9, 2024)
Announce this on X in separate posts scheduled 1 per day.
- ✅ Schedule an X post for the issues/discussions/pulls improvement.
- ✅ Schedule X post for GET all issues/pulls
- ✅ X post for the `?lines` improvement
# Avoid keeping API key queryparam in browser (november 10, 2024)
✅ If `apiKey` is provided and accept header indicates its a browser, make it a cookie and immediately redirect removing the apiKey
🎉 This makes things much safer. Now, people are prone to sharing their API Key on social media accidentally.
# `thread.html` security (november 10, 2024)
✅ It renders markdown. Ensure it's secure so it won't be able to inject any JS. This could be used to steal people's API key by sending them a link.
✅ Wonder the same for all raw code I render. Is it possible to hack it in a way so it renders HTML accidentally? can this maybe be prevented some other way?
# Update OpenAPI v1 (november 10, 2024)
- ✅ describe default maxTokens
- ✅ add `?lines` param
- ✅ add repo token count
- ✅ issues, discussions, pulls
- ✅ read all
- ✅ read one
- ✅ create one
- ✅ accept header support
# Track independent users and the repos they visit (november 10)
✅ Create an upstash KV store for `uithub-analytics`
- ✅ 1 counted set of owner/repo pairs
- ✅ 1 counted set of request count
- ✅ 1 just regular kv (hash -> ip, username, email, location) and key is hash(apiKey) or hash(ip). Email/username is only requested using API key if there wasn't a value yet
🎉 At least now we also can track independent users and whether or not they have authenticated.. Great to keep track of this KPI.
## Resolver in workflow (november 11, 2024)
✅ Use OpenAPI in `workflow.ts` => AI can do this based on OpenAPI.
✅ Ensure LLM token, model, and basePath is provided in query parameters or using cookie (and set to cookie).
✅ In workflow, ensure to respond with 402 if `model`, `basePath`, `llmApiKey`, and `messageResolverEndpointUrl` aren't found in queryparams or cookies.
❌ Ensure to actually call the endpoint and stream stuff back into a HTML page, so we can show it nicely a la bolt.new but then in the format of a git diff that streams into existence.
✅ For issue resolving as well as messaging, we must assume it done from the default_branch if not provided.
✅ We must name the branch in the right way based on params and based on transformer result
✅ The following URLs should call `resolveMessage`:
- ✅ https://uithub.com/owner/repo/issues/123/resolve
- ✅ https://uithub.com/owner/repo/discussions/123/resolve
- ✅ https://uithub.com/janwilmake/microflare/message/message%20here
👀🤔 Instead of streaming it to a browser, we could instead schedule a qstash message with callback. This would have the benefit that it happens in the background and if you interrupt it it doesn't stop. This may be nice but we still will need the original streaming endpoint, so let's not do this immediately and see if we'll really need this in the end. After all, we could still do this type of within githuq when watching new issues instead...
## Stream with correct response and tie to HTML (nov 12, 2024)
✅ In `resolveMessage`, respond with a `ReadableStream` so it can take long enough. Add additional status updates inthere.
✅ Ensure the final result in `/message` and `/resolve` returns a PR or branch URL on github
✅ Add `/message` and `/resolve` to openapi v2
✅ It seems to go wrong either in `push` or in `resolveMessage`. Let's see what we get in and out...
🔥 I can now `/resolve` any issue. test test test 🔥
# Further `push` api Improvements (nov 12, 2024)
✅ After some thought, improved interface: explicit lines for removal, join set-utf8 and set-base64, add setting via URL...
✅ Test creating new branch (target) from main with slight edit.
# New OpenAPI (nov 12, 2024)
V2 OpenAPI (available at /openapi-v2.html)
- ✅ push
- ✅ messageresolver
- ✅ message
- ✅ resolve an issue/discussion
# Ratelimits are hard (nov 12)
I've made the issue to PR flow work! However, I'm running into several hard to fix problems:
The first one is the ratelimits...
Per-minute ratelimits are quite strict:
- gpt 4o: 2,000,000 TPM
- claude-3-5-sonnet-latest 80,001 TPM
- claude-3-5-haiku-latest 400,000 TPM
For Claude especially this is problematic. I would say a context of at least 5000 lines is desirable, meaning we can not even do 2 iterations per minute.
TODO:
- ✅ request uithub JSON with no maxTokens set, return early for repos with more than 50k tokens showing the proper warning... also return early if there's a file with more than 4000 tokens.
- ✅ if we get ratelimited, ensure to view the ratelimit header and retry after it can be done again. if not available, use exponential back-off. Let's make a good implementation for this once and for all.
- ✅ send a status update if ratelimit is hit so the server knows
- ❌ request ratelimit increase for claude. **this wouldn't be a solution as everyone has a superlow ratelimit**
# The stream stops without saying anything.❗️❗️❗️ HIGHEST PRIO ❗️❗️❗️ (nov 12, 2024)
This is hard to figure out especially since we are doing multiple layers of streams:
- uithub (vercel)
- filetransformers (cloudflare)
- anthropic.actionschema.com (vercel)
- anthropic
We need to locate the problem by creating a more general test for a long stream.
Thoughts:
- could be vercel 1m default execution limit (although a stream should keep going right?)
- could be a cloudflare default firewall that stops requests after 1 minute. may need to use different type of syntax...
- could be something else
- maybe move push and messaging to a worker (also good for making it a smaller codebase)
TODO:
- ✅ read about streams on cloudflare. confirm doing a stream from cloudflare for 10 minutes
- ✅ tested in browser and found that it always stops consistently at 60s which happens to be uithub vercel.json limit. confirmed by docs: https://vercel.com/docs/functions/streaming-functions --> node.js can't stream infinitely!
- 🤔 as uithub relies on node.js environment to have lot of memory (the reason we need vercel) it will be hard to support this streaming endpoint from vercel at all. as i want to do more with cloudflare anyway, it makes sense to make that the root of uithub, with the only endpoint going to vercel being the one that requires the zip files. also anthropic.actionschema.com can be migrated, probably easily. this way we won't even really need to stream per se!
- 🤔 Another idea is to split up the vercel repo into functions through vercel.json. This is way faster!
- ❌ Move anthropic.actionschema.com to cloudflare
- ❌ Create an uithub cloudflare worker
- ❌ For the zip endpoint, use vercel.uithub.com
- ✅ Moved `/push, /resolve, /message` to separate files with `runtime: "edge"` enabled.
- ✅ Ensure we also get 401 page if needed
- ✅ Test functionality on localhost first
- ✅ Confirm it can now do >60s.
- ✅ Deploy and test prod
# Thread API + HTML (nov 15, 2024)
✅ Use cache.forgithub.com at `/owner/repo/issues|pulls|discussions`, returning the respective subsets of data. Also add the following URLs in the data:
- refreshUrl uses ?refresh=true
- ✅ markdown_url goes to /[n]?accept=text/markdown
- ✅ Add details_url to each thread going to uithub address
- ✅ Relevant code URL goes to https://getrelevantcode.com/[owner]/[repo]/issues|discussions/[number]/relevant-code
- ✅ Resolve URLs go to https://filetransformers.com/[owner]/[repo]/issues|discussions/[number]/resolve/resolve|plan|confidence respectively
✅ If accepting HTML, load issues.html, pulls.html, and discussions.html. Requirements: show the thread items, clickable, and add buttons for clicking the item itself or any other thing to go outwards.
✅ Ensure to have a canonical url setup if needed. Research this.
# New query params (nov 16, 2024)
✅ Add parameter `?yamlFilter` that takes a YAML string and filters the files based on that, returning the context in md/yaml/html/json just like regularly. (see summarize-folder for example in js). NB: 16kb is limit which probably would limit this param to about 1600 files max, which is fine.
✅ Make `?lines=true` default
✅ Cleaned up the code a lot
# Monetisation (nov 18, 2024)
- ✅ added pricing page
- ✅ improved user tracking
- ✅ added request counting in time windows
# Bringing it together (nov 21, 2024)
✅ Confirm getrelevantcode actually hits cache or not, and ensure its securely done such that you can't just get a private thread or code without api key.
✅ Confirm getrelevantcode can respond in JSON and confirm customContextUrl doesn't conflict in certain ways with other provided params such as owner/repo
✅ In getrelevantcode I want to get a sense of money spent. Ensure to add this as a response header (tokens used, cache hit, etc).
✅ On thread item, show relevant code. OUT: copy to favorite LLM...
❌ use githubreadproxy directly to get the default_branch if we need it.
✅ Hide pricing from landing for now.
✅ Button on uithub header to go to see all issues.. Hidden.
✅ In issuespage, also add 'message'. Add a banner to the page that this is a 'research preview'.
✅ Add `?cacheOnly` param to getrelevantcode, set to true if we're out of credits. Ensure it only shows 402 if getrelevantcode returns 402, and always request it.
✅ Test the ratelimit stuff I made previously. Set the ratelimit for unauthenticated high enough for it not to cause issues for individuals checking regular code subsets, for now.
✅ Respond with the requests left in headers and in a replacement in the HTML.
✅ Use uithub.cache for **default_branch**, it's faster. Test it. Fallback to `main`.
✅ Use `uithub.cache?simple` to get **repo size** beforehand. Deny request if repo is too big (determine max size for repos that time out). For now, 1GB is big enough!
✅ Check env variables!
✅ Fix analytics `mget`. Test it.
✅ Test thoroughly and deploy this new version.
✅ Just tested the github issues workflow. Wow, that works incredibly well! I can imagine it's going to be **radical** once we also get to see the confidence etc etc. Placing the comment, generating the code, making the PR (or deciding not to) should be easy choices that automatically get executed.
# November 22
✅ Add `?omitFiles=true` so i can save memory in cloudflare. Added it to openapi too
# Pricing Page and Stripe Setup (November 22, 2024)
- ✅ Think about pricing formula and reflect it on the pricing page
- ✅ Create TOS and privacy policy pages (includes storage of personal details)
- ✅ Login to Stripe
- ✅ Add Stripe subscription plan. Ensure it includes requirement to accept TOS / privacy policy upon payment.
- ✅ When ratelimit is exceeded and you are trying to do an action that costs money, ensure to always respond with a 402
- ✅ Ensure a 402 status creates a payment link and responds with 402.html
- ✅ Change 402.html so it's a very nice sales page saying become a more productive dev - 10$/month [pay] CTA
- ✅ Add stripe callback to stripe and retrieve webhook secret
- ✅ Create a function to create a paymentintent with metadata attached (github ID required)
## `/buy` (nov 22, 2024)
- ✅ Should show 401 if no github ID
- ✅ Should create payment link in the background based on if you're premium already or not
- ✅ Should show 402 in the same way
- ✅ Pricing page link goes to `/buy` too
- ✅ Show credits in header. This gives people awareness that payment will be required.
- 🟠 Click goes to `/buy`
# Stripe Subscription Persistence (nov 23, 2024)
- ✅ in getrelevantcode, repsond with i/o tokencounts
- ✅ Deduct the right amount
- ✅ Add stripe keys and deploy
- ✅ Implement callback correctly doing nothing except logging
- ✅ Make a subscription to test and log
# Additional credit logic (nov 23, 2024)
- ✅ Send back the token usage from `getrelevantcode`, calculate that into price, and make it matter for large requests.
- ✅ 402 page logic to show the right stuff and create the right link
- ✅ Create a test fn that retrieves all subscriptions. Look if the github userId is there. ~~If not, we need customerId as well maybe, or there is a way to tie it to the customer when creating the payment link~~ **it is tied to the subscription now. perfect**
- ✅ Ensure there's a path to cancellation and this is documented ~~maybe its as easy as adding a link to the pricing page~~ **yes, it's that easy**!
# Stripe Webhook (nov 23, 2024)
- ✅ Implement `stripe-webhook.ts` for adding a subscriber and removing (cancel or pause events) setting it to the KV
- ✅ logic in webhook that puts the additional credits onto the user if it was a one-time payment.
- ✅ Correct logic adjusting creditsUsed if user has credits and spent the monthly allowance already
# small improvements (december 6, 2024)
- ✅ removed need for loading the repo (we don't need to know the default branch because we get commit sha, and the size is not important enough to block)
- ✅ uithub now works with exact commit shas too
- ✅ tried another approach caching the zip uncompressed at cloudflare but seems very expensive as it's a single write per file. storing it as a JSON object will hit memory bounds too easily and/or complex. left research at https://github.com/gildas-lormeau/zip.js/issues/545
# December 10
solved a small bug where https://uithub.com/ai16z/eliza/ got 404. added a redirect to https://uithub.com/ai16z/eliza
# January 9, 2025
✅ Update `uithub` to improve size: Have it also count files, total text characters, and lines count. Also ensure its possible with a param to only get the size.
# March 12, 2025
- ✅ Ensure to show logged in
- ✅ Ratelimit without login 10x per hour
- ✅ With login throw 'payment required'
## Ratelimit, auth, and monetisation (2025-03-15)
- ✅ Remove stripe stuff. Full sponsorflare support
- ✅ Create ratelimit DO. ip-based ratelimit for unauthenticated and underfunded users (balance < -1.00): 25 requests per hour unauthenticated, 50 per hour if signed in.
- ✅ Make it work again. getting 401 now
- ✅ Remove old 402 page
- ✅ Ensure to have a 404 page with login button and privacy policy and terms, that support private access, even if you are already logged in.
- ✅ Ensure to have a 429 page with counter with either login button and privacy policy and terms, or if logged in, it shows balance and link to sponsor more.
- ✅ Fix issues regarding login cookies etc, ship new version sponsorflare that makes debugging easier
## Make it work sortof (2025-03-18)
- ✅ Proxy to https://zipobject.vercel.app directly. For now the api key stuff isn't very reliable yet, so it's best to leave this out for now. At a later stage, uithub is kind of a project on top of zipobject and should be connected accordingly.
- ✅ Show user in html
- ✅ Add message inputbox at bottom leading to https://chat.forgithub.com/owner/repo?q={input-urlencoded}
- ✅ forgithub is too vague for ppl. clear it up
- ✅ remove pricing entirely for now.
- ✅ Remove old usernames (cfa, k)
- ✅ landing: add mnemonic device section
- ✅ Cleaned up files a lot
# Make it as good as uithub v1 (2025-03-20)
**Artem / Murzin**: uithub UI can look better, filters must be easy to use (1% uses api only). More important button more prominent.
🤔 After brainstorming and experimenting with the UI, it became clear to me that, even though I initially thought it was cool to switch between a UI and raw text (md, json, yaml), the HTML interface should probably be the leading way to do get to your context. It's tricky that in the HTML interface I want to show different formats (JSON, YAML, or Markdown) while still handling the filetree, size, and datapoints as independent datapoints, because they become part of the interface. In the end it's likely better to:
1. Get the tree with sizes separately via https://tree.forgithub.com as well as repo metadata. As filetrees can be cached for a day with ease, this is going to be incredibly fast.
2. Show the view.html immediately after that, including perfect SEO
3. In the UI load the desired data via a fetch upon initialisation. This allows to already change filters while it's still loading, and faster navigation through large codebases. It also allows LLM filter which may be slow. The loading indicator is the desired UI here.
# Make it look good version (2025-03-20):
- ✅ Simplify HTML further
- ✅ Implement VSCode-style interface
- ❌ Tried removing flicker but failed.
- ✅ Get tree and content from https://zipobject.com in 2 separate api calls
- ✅ Calculate current tokens manually for now, simply doing `string.length/5` ^^
- ✅ Create {{template}} replacer and serve `vscode.html` with data
- ✅ Fix styling issues
- ✅ Render tree based on `window.data.tree` (folders with sizes, no files)
- ✅ Add 'REDIRECT_DOMAIN' var (Set to new.uithub.com for now) and redirect it to context.forgithub.com if landed on the worker. Great thing is: the static stuff will still be served from it!
## POC UIT (2025-04-02)
In the POC I want to focus on processing GitHub archives in this 4 step and making it available through the new uithub interface. The components needed are:
- `uithub.tree`: zip to tree-json-sequence
- `uithub.ingestzip`: zip to content-json-sequence
- `uithub.search`: apply standard search-filters (jsonseq->jsonseq)
- `uithub.merge`: turn potentially multiple json sequences into 1
- `uithub.outputmd`: double stream json seq into a markdown with tree with sizes first, content last.
- `uithub`: couples the above based on URL path with filters and presents resulting tree and md in HTML, adding ratelimiting, authentication, and monetisation.
- ✅ come up with the right JSON Sequence datastructure with minimal loss of information of all origin formats. see what I had in zipobject and zipobject.tree
- ✅ implement `ingestzip`
- ✅ implement search
- ✅ lookup filters definitions zipobject
- ✅ create definition in markdown for that with appropriate context
- ✅ generate `jsonseq->jsonsec`
- ✅ implement merge
- ✅ spec
# Output zip (2025-04-04)
✅ Implement `outputzip` to easily go from zip to zip in a streaming fashion
✅ Confirm its fast and immediately streams the zip through 2 layers
# FormData POC (2025-04-04)
- Make all endpoints accept POST with body without adding too much complexity. Keep definition leading
- ✅ ingestzip
- ✅ search
- ✅ outputzip
- ✅ Implement clever URL logic on this: `/[domainOrOwner]/[repoOrId][.ext]/tree/[(shadow)branch]/[basePath]`. See `convention.md` for how exactly.
- ✅ I can now use `main.ts` for the markdown chain to go from any `storage --> formdata -> search [-> transform] -> zip`
- ✅ it works from and to zip with direct streaming WITH BUN 🎉
- ❌ with search in between it breaks now
- ✅ try search via post first via node js fn
- ✅ figure out if search has proper error handling
- ✅ if search works, see if 3-step pipe works.
- ✅ see if it also works in prod
- ✅ see if it als works for bun
- ✅ Goal: https://uit.uithub.com/oven-sh/bun.zip/tree/main?basePath=/docs would immediately start streaming the zip.
- ✅ Improve url pattern more.
- ✅ Goal today: visit https://pipe.uithub.com/oven-sh/bun instantly get the first 50k tokens streamed back.
# Refactor (2025-04-05)
- ✅ Rethink the API so search and path search is clearly separated and it matches UI better
- ✅ Tie uithub to the new formdata version.
- ✅ Ensure tree and content load simultaneously
- ✅ Figure out why its sorta slow now? Maybe should still do stream of the content via separate frontend-fetch? Or maybe i can make it fast enough still. Speed is kind of the #1 reason for uithub to exist.
# Improved tree (2025-04-06)
- ✅ Ensure tree is cached using KV and `stale-while-revalidate` works too.
- ✅ Tree must load when branch wasn't provided
- ✅ Fill total repotokens from `tree.__size`
- ✅ Make dev easier with service-binding helper to use either fetch or binding. When bindings aren't connected would just fallback to regular fetch.
- ✅ 🎉 Confirm bun is fast now (loads in under a second)
- ✅ Ensure nav menu is open when we're at a specific path
- ✅ showFiles toggle must be localStorage-reminded
- ✅ Ensure it shows inactive files (based on path) greyed out
- ✅ Ensure it shows the currenly active basePath highlighted (purple).
- ✅ Add copy button to tree nav that gets full file tree (JSON string is fine for now).
- ✅ In `uithub.pipe`, ensure 'basePath' follows through correctly. make it more flexible with `/`
# UI/UX (2025-04-06)
- ✅ Added markdown support
- ✅ Fix maxTokens input filter
- ✅ Fixed problematic misconception (token-count): https://x.com/janwilmake/status/1895375026939142292
- ✅ Menu always open by default
- ✅ Improved markdown suffixed message
# Add profile section & auth (2025-04-06)
- ✅ Make `profile.js` that gets activated after clicking the profile icon
- ✅ Show 'unlock premium features' with a button 'login with github'.
- ✅ After sign-in, show balance and donate button.
- ✅ List features that are unlocked after you donate.
- ✅ create and connect uithub client, test login
- ✅ fix ratelimiter
THIS IS KEY TO STARTING THE MARKETPLACE
# Fix URL structure (205-04-07)
✅ Bug: owners CANNOT. BUT! repos CAN contain dots, so ext cannot be part of id! Rather, make it part of page, e.g. tree.json, tree.md, etc. This is precursor to fixing https://new.uithub.com/brunabaudel/brunabaudel.github.io
# Auth works and is fast (205-04-07)
- ✅ When logged in i get `Tree errorFailed to fetch ZIP file: 404 Not Found` for public repo
- ✅ Pass on auth down the chain.
- ✅ Confirm going through both public and private repos works.
# Refactor: direct link to pipe (2025-04-08)
✅ To make it all run a bit easier, uithub should directly call the pipechain via `urlPipe`.. Refactor this so it does.
# Provide default branch for github repos (2025-04-08)
✅ Non-'main' default-branch repos should be navigated properly. For this we must somehow retrieve the default branch or head sha without slowing things down. This info is available in the zip as the first folder is always: `[repo]-[branchOrSha]`. Let's get this in a response header from tree.
✅ This fixes navigation from https://new.uithub.com/brunabaudel/brunabaudel.github.io and any other master default branched repos.
# Mobile warning (2025-04-08)
✅ On mobile user-agents that aren't found to be tablets, it redirects to `mobile-not-supported.html`. This way stuff remains pretty.
# `search.js` (2025-04-08)
Get the search spec and remake `search.js`. Can be done using prompt completely.
- ✅ Populate search from and to URL
- ✅ Changing URL contents should add them to URL history but not refresh page yet
- ✅ Enter in any field should apply URL
- ✅ When history has been altered but page not refreshed yet, an 'search' should appear.
- ✅ When there are search filtered applied. A 'clear' button should be enabled that clears all search filters.
- ✅ Searchbar always have path filters open.
# Search/pipe backend (2025-04-08)
- ✅ Test glob patterns and ensure exclude is applied after include
- ✅ Test search
- ✅ Test match Whole word
- ✅ Test case sensitive
- ✅ Improve maxFileSize handling
- ✅ bug basepath filter https://new.uithub.com/janwilmake/uit/tree/831232048291a7f2c96821d020d7761d293ede98/uithub (fixed)
# UI/UX (2025-04-08)
vscode.html
- ✅ `maxTokens` input should update on submit
- ✅ allow setting `maxFileSize` similarly
explore.js
- ✅ Navigation path-clicks should keep rest of query
- ✅ Navigation greying out isn't fully correct right now.
- ✅ It gets expanded too much if you're deeply nested, e.g. on https://new.uithub.com/facebook/react/tree/336614679600af371b06371c0fbdd31fd9838231/compiler/packages
# 'Add to README' button (2025-04-18)
✅ Fix https://github.com/janwilmake/forgithub.badge
✅ Make it an API as well that takes a github-related URL, and responds with markdown for a badge for it, and also provides the github URL of the page to edit.
Sha url can't be used for edit link, must be branch!
✅ Then embed that in the `vscode.html` so it copies the markdown and opens the readme in edit-mode, so you can paste it.
✅ For now, remove chat with LLM. Not worth it for now.
🤔 Learned that github requires any image to load within a few seconds (and caches it); too slow makes it unusable. To solve this (and to also solve it for og:image's) let's focus on performance in the first place, not caching per se. For this, it's fine to have an outdated number of tokens, but it also needs to be a fast experience in the first place for it to be usable!
# OLD UITHUB improvements (2025-04-19)
- ✅ add "Add to README" to old uithub too
- ✅ connect `forgithub.badge` tokencount with old uithub
- ✅ Add top 500 with small size to old uithub too (landingpage)
# Go to market V2 (2025-04-19)
- ✅ Fix https://activity.forgithub.com
- ✅ let's make this update every 24 hours: https://popular.forgithub.com
- ✅ let's add top 500 to the landingpage!!!
- ✅ Also add to the githuq.com landingpage
- ❌ Let's add questions to each: https://questions.forgithub.com
- ❌ Make chat.forgithub.com fast
# 2025-04-21 - Feedback tl;dr - open questions to make the marketplace work:
- ✅ Error handling sucks. how to improve?
- ✅ Need standardized way to charge
- ✅ URL chain auth pass sucks.
- ✅ No easy getting started/docs
# 2025-04-21 - solve architectural issues
- ✅ Solved all major problems found in initial feedback (see [ADR](ADR.md))
- ✅ Make `GETTING-STARTED.md` guide
# 2025-04-25 (FEEDBACK)
- ✅ Improved plugins tab
- ✅ Improved overal layout
- ✅ Improved search tab
- ✅ 'No results' warning: 'No results. Please check your filters'
- ✅ Tab should be localStorage reminded, not populated from URL
- ✅ Add download button
- ✅ Made quick draft for outputjson
- ✅ Removed YAML Button for now
- ✅ Test `isRegex`
- ✅ Added 'copy as curl' button
- ✅ Test login 401 flow after hitting ratelimit.
- ✅ Identified private repo problem. Fix tree error!!!
- ✅ Fixed branch bug when logged in
- ✅ Private repo not working; https://uuithub.com/janwilmake/forgithub.activity. Add repo scope!
- ✅ Make a schema for FAQ; answers would be instantly answerable by LLM
- ✅ Create default faq and FAQ.json for `uit` which, currently, just inherits from the default.
# 2025-04-26
- ✅ PERFORMANCE (Try filter on path and extension(binary) early). If a file can be filtered out without loading the file itself, that must be done! I want things to be FAST. it should especially be fast skipping over files we don't need such as binary and files and files for which the path/size doesn't match.
- ✅ Try https://uithub.com/sam-goodwin/alchemy/tree/main/alchemy-web/docs vs https://uuithub.com/sam-goodwin/alchemy/tree/main/alchemy-web/docs. Must be AS FAST!!!
- ✅ Add `FormData` to type selector and default to markdown
- ✅ Ensure the path pattern for domains can be variable on a per-domain basis
- ✅ Added concept for lists
- ✅ For uithub, make routing of plugins work. api plugins are basically ingest plugins. Rename that.
- ✅ Added search filters and replaced minimatch with picomatch for creating precompiled regexes
- ✅ Omit binary based on extension too
- ❌ Implement https://en.wikipedia.org/wiki/Boyer–Moore_string-search_algorithm
- ✅ Takes 2 minutes for bun. speed can likely be improved down to about 15-40s. see https://claude.ai/share/d4059e61-7ab8-404a-b5cd-f4dc2823101c
# 2025-04-28
- ✅ Clean up `ingestzip`; Added proper binary filter on paths as well as content.
- ✅ Added `omitBinary` to `uithub` requests for non-zip responses. For zip responses, binary is not omitted anymore.
- ✅ Added custom `x-filter` FormData header that provides information about files that got filtered out
- ✅ should include entire file structure, including files where content got omitted
# `.genignore` (2025-04-30)
✅ I've added `genignore.ts` to `ingestzip` so it always first finds genignore or uses the default.
✅ If `genignore=false` is provided, should disable default or configured genignore.
✅ If `excludePathPatterns` is provided, these are added to the patterns (duplicates removed)
✅ Update OpenAPI spec
✅ exclude patterns should not include ones starting with # or if its an empty string, trimmed.
✅ In frontend, for some reason. it is rendering it as html. improved escape functionality
✅ In frontend add `genignore=false` checkbox titled `disable genignore`.
✅ In frontend, add button `Create .genignore` that does the same as README button, but for `.genignore`
# `.genignore` again (2025-05-01)
✅ Research the `.gitignore` specification and compare that to VSCode specification for `files to include/exclude`. Determine how `.genignore` should work, and write that into a spec at `uit/specification/genignore.md`
✅ Create a nice landing for genignore.com that explains the concept using that file. Also include the fact that repomix and gitingest also had introduced/discussed similar concepts, but a universal one is more desirable.
✅ DM them. This should be enough to make `.genignore` succeed as a standard!
🤔 How are files in the zip sorted? How to get the `.genignore` asap, then use it to filter files? How will it work if I have multiple repos and am ingesting them as a single FormData stream? The `.genignore` won't be in the root anymore. Generally I just need a way to get config files from any zip, beforehand, without making it slower.
✅ Confirm that `.genignore` works like desired (or fix) including the early returning in the right moment.
❌ Try to improve the `excludePathPatterns` so negations work as expected, so https://uuithub.com/janwilmake/forgithub.popular?excludePathPatterns=*&excludePathPatterns=%21README.md works. Potentially, people wanna have only a few specific files in their context, which should be specified like `*.*\n!README.md`. **It's hard. Let's think about this.**
# Nailing omni-compatible navigation (2025-05-01)
🤔 Make it possible to see search filters in tree as well by moving this logic to the backend. It's likely best to stream the formdata after search to `uithub` directly so i can build/return the tree instead of `ziptree`. This way I know which files got filtered using `x-filter`.
✅ `outputmd` should take FormData from body rather than just from URL
✅ The `output*` service should be called using `repsonse.body.tee()` in `uithub`. We use the structured FormData output to generate the tree in a helper utility function.
✅ Ultimately, the tree datastructure would be `{ [segment]: { size: number; filtered: boolean, children: this }}`
✅ Pass StandardURL data to HTML `window.data`.
✅ Add `x-filter` and `x-error` type safety to `multipart-formdata-stream-js`
✅ Create `buildTree` (take inspiration from: `uithub.ziptree`)
✅ Get `defaultBranch` in another way. This is github specific, so maybe should be done in the `github.ts` router?
✅ Apply `StandardURL` data and new tree datastructure in frontend. Ensure we use it to navigate!
✅ `explore.js`: gray out based by comparing final paths with filetree via `string[].includes`. For this we need the final tree as structured data as well.
# package resolving (2025-05-02)
This is what needs to be done for this:
1. ✅ create `ingesttar` and npmjs domain binding to `uithub`
2. ✅ get `npmjz` module resolution as I had before with function to also get all versions based on package.json/npmrc.
3. ✅ ensure the above allows only finding packages that are specified, not subdependencies
4. ✅ deployed at https://npm.forgithub.com
5. ✅ added to `plugins.json`
# `ingest` plugins (2025-05-04)
- ✅ Make `ingestjson.uithub.com` so all the apis make sense! Make it adhere to `{files:{[path]:{content}}}` and if that format isn't found, take first key as `{key}.json`
- ✅ finalize routing logic for ingest plugins
- ✅ ensure ingestjson has content-type + boundary such that it doesnt error out. Test locally.
- ✅ ensure npm.forgithub.com works
- ✅ ensure cache.forgithub.com works
# Github URL structure (2025-05-04)
- ✅ Keen out the URL structure of GitHub and which URLs I can support easily
- ✅ Improve github url parsing `github.ts` for issues/pulls/etc I need to alter what the basepath becomes.
- ✅ Ensure https://uuithub.com/facebook/react/issues/17473 makes `17474` the basepath of the source that is `issues`
- ✅ Remove actions.forgithub.com for now
# `npmjs.com` domain (2025-05-04)
- ✅ create `ingesttar` and npmjs domain binding to `uithub`
- ✅ add npmjs.com into domains; `npmjs.ts` should route to the appropriate package
- ✅ Make `domains.json` function
- ✅ confirm I can get packages by changing from npmjs url structure to uuithub.com/npmjs.com/...
- ✅ determine npmjs.com url structure and what the sources should be for that through some table
- ✅ http://localhost:3000/npmjs.com/package/reactatron should work
- ✅ navigation from http://localhost:3000/npmjs.com/package/reactatron/v/0.0.3 should work.
- ✅ make it work without version (resolve dist/latest)
- ✅ make filters work. seems non-responsive
- ✅ just redirect npmjz.com to uuithub.com/npmjs.com/...
## /CONTRIBUTING.md
# Contributing to UIT
Thank you for your interest in contributing to the Universal Information Terminal! This document provides guidelines and information about contributing to this project.
## Code of Conduct
By participating in this project, you agree to maintain a respectful and constructive environment for everyone.
## How to Contribute
### Reporting Issues
- Check if the issue has already been reported
- Use the issue template when available
- Include detailed steps to reproduce the problem
- Specify your environment details
### Feature Requests
- Clearly describe the feature and its use case
- Explain how it aligns with the project's goals
- Consider how it would be implemented
### Pull Requests
1. Fork the repository
2. Create a feature branch (`git checkout -b feature/amazing-feature`)
3. Commit your changes (`git commit -m 'Add some amazing feature'`)
4. Push to the branch (`git push origin feature/amazing-feature`)
5. Open a Pull Request
### Development Workflow
1. Set up your development environment following the instructions in the README
2. Make your changes
3. Ensure tests pass if applicable
4. Update documentation as needed
## Attribution
Please note that while this project is licensed under MIT, we strongly encourage attribution as outlined in [ATTRIBUTION.md](ATTRIBUTION.md). When contributing code, your contributions will be covered by the project's license, and we appreciate maintaining the attribution notices in code files.
## Style Guidelines
- Follow the existing code style and patterns
- Include appropriate comments
- Write clear commit messages
- Update documentation for any user-facing changes
## Documentation
Good documentation is crucial for this project. If you're changing functionality, please:
- Update relevant README files
- Document new functions/modules with clear descriptions
- Add usage examples when appropriate
- Include code header comments as shown in the template
## Questions?
If you have any questions about contributing, please open an issue or reach out to the maintainers.
Thank you for helping make UIT better!
## /GETTING-STARTED.md
# Running locally
Prerequisites:
- [wrangler](https://developers.cloudflare.com/workers/wrangler/install-and-update/)
Installation:
- Run uithub
```
git clone https://github.com/janwilmake/uit.git`
cd uit/uithub
wrangler dev
```
- Run sponsorflare
```
git clone https://github.com/janwilmake/cloudflare-sponsorware.git`
cd cloudflare-sponsorware
wrangler dev --port 3001
```
Ensure to setup `.dev.vars` correctly (copy `.dev.vars.example` and add your GitHub PAT (create one [here](https://github.com/settings/tokens))).
# Developing a new plugin locally
Add your plugin to [plugins.json](uithub/public/plugins.json) and explore it through the interface. Should work with a remote plugin as well as a local one on another port, e.g. `http://localhost:3002`.
# Running a new plugin directly from the hosted interface
This isn't possible yet, but I'm working hard to realize this. Ideally you'd configure the `dev` plugin from the interface with a endpoint URL that is submitted as a shared cookie. This URL must be a remote URL that follows the FormData to FormData spec or is in API format. When visiting the `dev` page (https://uithub.com/[owner]/[repo]/dev/[branch]/...) the plugin provided would become accessible.
🤔 Instead of having to develop `uithub.murl` immediately, it might be faster to build this out quickly and discourage local development altogether. However, having uithub.murl is gonna be great!
## /GROWTH.md
# OSS Funding
✅ Applied to nlnet.nl; https://news.ycombinator.com/item?id=43769482
https://x.com/dok2001/status/1910591729968759240; consider https://www.cloudflare.com/en-gb/lp/project-alexandria/
Consider other OSS funds https://claude.ai/share/4b9c6228-fb0c-4eee-9a75-84c09cc1aa99
# Bounty and Open Source Repo List
Put together a list of lists. Every list can be sourced by someone else and turned into a list of URLs of repos with a simple parser.
There aren't any lists on github of repos with bounties
Bounty program Apps:
- https://www.bountyhub.dev
- https://algora.io
- https://www.boss.dev/issues/open
However, we can also assume any open source repo with funding or revenue would give us money if we speed up their development.
- Scrape the bounty program app apis or websites and get to a list of owners and repos with that
- Scrape https://www.ycombinator.com/companies/industry/open-source and find the belonging github owners
- Scrape https://www.ycombinator.com/companies (5000+) and figure out if they have a github repo and have the owner.
- Scrape github's biggest open source repos and figure out if they belong to companies with revenue or funding.
Obviously this will take some time to get to this in an automated way... But it's much faster to start manually creating a single list in a repo readme: https://github.com/janwilmake/awesome-bounty-repos
From here, we need to get to determine which repos are properly suitable for us to navigate in and build proper issue validation.
After I nailed this these filters, I can start cloning repos and solving issues in my cloned repos, and make PRs.
If I add a feature with a "TIP JAR" to every PR I made with a suggested price, the algo can start optimising maximising profit and minimising cost, in other words maximising EV.
Maybe this is too ambitious still, because the repos are actually very large, issues can be complex, and priorisation is hard. Maybe it's better to first focus on my own code of which I know much better how to solve issues.
## /LICENSE.md
MIT License
Copyright (c) 2025 Jan Wilmake
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
## /README.md
# UIT - Universal Information Terminal
- [](https://uuithub.com/janwilmake/uit)
- [X Thread](https://x.com/janwilmake/status/1915774892500300281)
- [Hackernews Thread](https://news.ycombinator.com/item?id=43793986)
UIT is a library for **performant, modular, low-memory** file processing at scale, in the Cloud. It works by offering a 4-step process to gather a file hierarchy from any desired modality, apply filters and transformations, and output it in any desired modality.
- **performance**: speed is of essence when navigating and searching through large amounts of data
- **low-memory** by applying streaming and parallelization we can run this in low-memory environments such as Cloudflare workers
- **modular**: modularity is beneficial because by making it composable we get a clear high-level overview of all building blocks. also, not all building blocks can be ran in the same runtime or location.
> [!IMPORTANT]
> This is an early pre-release. See [TODO](TODO.md) for current challenges. Try the demo at https://uuithub.com
UIT has come about after many iterations of the platform of [uithub](https://uithub.com), which started as a simple node-based parser of zipfiles. While building more and more features and add-ons, I found myself limited by the memory a lot as I was not streaming enough, and going back to JSON too early (because using the Streams API is tricky!). Thus, as features and complexity grew the need was born to create a more modular extensible architecture with good serverless practices in mind.
`FormData` has a long history [[RFC 1867 (1995)](https://datatracker.ietf.org/doc/html/rfc1867)] [[RFC 2388 (1998)](https://datatracker.ietf.org/doc/html/rfc2388)] [[RFC 7578 (2015)](https://datatracker.ietf.org/doc/html/rfc7578)] and is deeply embedded into the web. It offers an excellent way to serve multiple files, binary and textual, over a single request. Although `FormData` does not support stream-reading directly from `Request` and other Web Standards yet, UIT leverages the fact that intermediate results can be read using the [Streams API](https://developer.mozilla.org/en-US/docs/Web/API/Streams_API) using [multipart-formdata-stream-js](https://github.com/janwilmake/multipart-formdata-stream-js).
UIT cleverly modularizes filters and transformations on file hierarchies by providing an elegant way to combine multiple UIT 'modules' together to get to a final result. Every UIT 'module' can apply path filters, content filters, and content transformations, to change the files in the file hierarchy, all while streaming, and even merge multiple file hierarchies together in the blink of an eye.
# UIT Modules
So far, UIT provides the following modules that can be combined to create powerful file processing pipelines:
**ingest modules**
- [**uithub.ingestzip**](./uithub.ingestzip) - Ingests and processes ZIP files into normalized formdata format
- [**uithub.ingesttar**](./uithub.ingesttar) - Ingests and processes gzipped tarfiles into normalized formdata format
- [**uithub.ingestjson**](./uithub.ingestjson) - Ingests and processes JSON into normalized formdata format
**transform modules**
- [**uithub.search**](./uithub.search) - Provides search capabilities across file hierarchies
**output modules**
- [**uithub.outputmd**](./uithub.outputmd) - Transforms and outputs data as markdown files
- [**uithub.outputzip**](./uithub.outputzip) - Packages processed data into downloadable ZIP archives
**helpers**
- [**uithub**](./uithub) - Brings several modules together, pipes through them, and shows in authenticated HTML interface.
- [**uithub.otp**](./uithub.otp) - Source proxy that generates an OTP to minimize secret exposure to other modules.
Each module is designed to perform a specific step in the UIT 4-step process (ingest, filter/transform, merge, output) while maintaining performance and low memory usage.
It is important to note that each of these modules can be independently hosted as a cloudflare worker, but the spec doesn't require it to be hosted on Cloudflare per se, you can also host UIT modules in other runtimes, as long as it's compliant with the [UIT Protocol](#uit-protocol)
Please also note that above diagrams showcase many modules that haven't don't exist yet, but could be beneficial to exist. By Open Sourcing UIT, I hope to empower developers to add the modules they need.
# UIT Protocol
The UIT Protocol is the convention that characterizes any UIT module. As can be seen in the diagrams above, any UIT module must be one of these 4 module types:
- **ingest module** - streams any datastructure into a FormData stream
- **formdata transform module** - applies filters and transformations on files in a streaming fashion while in the FormData 'modality'.
- **output module** - streams a FormData stream into any desired datastructure
The only formalized convention/protocol you need to understand to create a UIT module, is which FormData headers UIT modules work with. These FormData headers can be divided into standard and non-standard (custom) headers:
## Standard FormData Headers
| Header | Description | Required |
| ----------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------- |
| **Content-Disposition** | Contains `name` (should equal pathname) and `filename` (should equal pathname) | Yes |
| **Content-Type** | Specifies the MIME type of the data | No |
| **Content-Length** | Indicates the uncompressed size of the data | No |
| **Content-Transfer-Encoding** | Specifies how the data is encoded:
- `binary` (required for binary files)
- `8bit` (recommended for text-based/utf8 files)
- `quoted-printable`
- `base64`
- `7bit` (default) | No |
## Non-Standard (Custom) Headers
| Header | Description | Format |
| --------------- | ----------------------------------------------- | -------------------------------- |
| **x-url** | Specifies the URL that locates the binary file. | URL string |
| **x-file-hash** | Stores the hash of the file | Hash string |
| **x-error** | Indicates processing error in the pipeline. | `{plugin-id};{status};{message}` |
| **x-filter** | Indicates a file got filtered out. | `{plugin-id};{status};{message}` |
Important:
- In some cases it may be desired to omit the binary data and only leave the URL to locate the file.
- On error in a module, the original incoming file-content should be preserved. If encountered, shouldn't be filtered or processed, so we can see errors for every individual file, where they happened, and with what file input.
- On filtering in a module, the `FormData` can be passed on with `x-filter` but without content
- On renaming and/or transforming, `x-filter` does not need to be aplied.
# Contributing to UIT & Plugin System
UIT aims to be a convention to streaming, filtering, and transforming binary and textual file hierarchies in the Cloud, and maintains a curated list of first-party and third-party libraries that can be included into any UIT data-transformation flow.
As a first step I aim to create a plugin system that allows doing file filters and transformations with ease from the uithub UI. For intended plugins, check out [plugins.json](uithub/public/plugins.json) and [the spec](uithub/public/plugins.schema.json).
[The multipart parser](https://github.com/janwilmake/multipart-formdata-stream-js) is designed to handle all `FormData` headers, including any non-standard ones, and can be a useful libary to create FormData filter/transformers. It extracts them from the raw header lines and makes them available in the Part object. The library also maintains the original `headerLines` as part of the parsed data structure.
Please open a discussion, issue, pull request, or [reach out](https://x.com/janwilmake) if you want a new module to be added to this list or have any unmet requirements. To create your own plugin, follow the [GETTING-STARTED.md](GETTING-STARTED.md) and [CONTRIBUTING.md](CONTRIBUTING.md). UIT is also looking for sponsors.
# Links
- [TODO.md](TODO.md)
- [BACKLOG.md](BACKLOG.md)
- [CHANGELOG.md](CHANGELOG.md)
- [ADR.md](ADR.md)
- [GETTING-STARTED.md](GETTING-STARTED.md)
- [CONTRIBUTING.md](CONTRIBUTING.md)
- [ATTRIBUTION.md](ATTRIBUTION.md)
- plugins [[html]](https://uuithub.com/plugins) [[json]](uithub/public/plugins.json)
# Community & Related work
- https://github.com/cyclotruc/gitingest
- https://github.com/yamadashy/repomix
- https://repoprompt.com
- https://github.com/wildberry-source/open-repoprompt
- https://dev.to/mnishiguchi/building-a-local-ai-friendly-code-viewer-inspired-by-uithub-24ll
- https://github.com/janwilmake/zipobject.vercel - earlier version intended to replace uithub, got too complex (not modular, hard to debug)
# License and Attribution
> [!IMPORTANT]
> MIT will be added after official launch
UIT is licensed under the [MIT License](LICENSE.md). While the license only requires preservation of copyright notices, we kindly request attribution when using this project. See [ATTRIBUTION.md](ATTRIBUTION.md) for guidelines on how to provide attribution.
~ Being made with ❤️ by [janwilmake](https://x.com/janwilmake)
## /TODO.md
`&basePath=`
# TODO
1. ✅ make it usable and gather early feedback (silently open source it) (TARGET: april 8, 2025)
2. ✅ Rename all of zipobject to uithub. uithub means 'universal information terminal hub'
3. ✅ Let's show some ppl/ais and figure out what to do next! (did this until april 20, 2025)
4. ✅ Deploy to uuithub.com
5. ✅ Announcement (Friday, april 25th, 6PM)
6. ✅ Nailing omni-compatible navigation
7. ✅ ingest plugins
8. ✅ npmjs.com domain
9. ❗️ Navigation
10. ❗️ Stability. Goto prod?
11. Fix ingestzip for wikizip
12. Merged source
13. xymake zips
14. `.genignore`
15. `context.json`
16. agent-friendly middleware
17. Implement `uithub.otp` and use it in `uithub.ingestzip`
18. Implement `monetaryurl`, pluggable into `sponsorflare` and `stripeflare`, and use it everywhere
^^^ This can still take several weeks to make good. Target: end of may ^^^
# Navigation improvements
- In router, calculate source URL so it is used as button
- On frontend, use fork icon for source and provide rounded token number
- In router, allow providing `more tools` url. if not, no 'more tools' button.
- 🤔 It'd be interesting to be able to determine all possible pages for a particular domain, or at least, some. For example, to easily navigate up from a repo source to the owner source. This may even be a part of the navigation, rather than repos.
# Stability
- In `outputmd`, add the params that were used to search as frontmatter. Also, add warning to the top if tokens were capped.
- When generating full markdown-tree, also get token size for each file/folder, and show the size on folders
- `outputjson` should take request.body as possible way of input (JSON output should work)
- `?accept=multipart/Form-Data` should render HTML
- Ensure we get the default branch for any github repo from a KV.
- We don't get the size of filtered out files. Ensure we still get this (keep content-size header alive). When filtering out files in `ingestzip`, ensure original filesize is preserved.
- `search.js`: basepath should show up on search to easily remove
- Bug with spaces: https://x.com/janwilmake/status/1898753253988253946
- Add https://www.simpleanalytics.com event if free plan (**yes, is free**) (see: https://docs.simpleanalytics.com/events/server-side)
🤔 Review it: What prevents me from hosting this at uithub.com with uuithub.com being the public staging environment that can be unstable? I just made it possible to vary between `wrangler deploy` and `wrangler deploy --production`. Prod can still break when editing sub-services though, so this is difficult.
# uithub as API product
- Provide programmatic way to login and get API key, and document this in the OpenAPI. It's probably good to add this into sponsorflare as well. --> Follow MCP recommended oauth 2.1 spec!!
- Provide programmatic way to retrieve usage and show this in a table on per-month basis as well as last 14 days on a per-day basis in a graph.
- Provide ability to create/rotate api key, and ensure the api key is not the same as the key you login with, but a key specifically made for API use.
After this is there, this'd be a great thing to show to people, as a minimal example of how to build a paid API with Cloudflare.
# Ingestzip wiki broken
All files are empty - it may be the inflate/deflate method? https://ingestzip.uithub.com/https://wikizip.forgithub.com/Netflix/Hystrix/wiki
BONUS: `ingestgit` would stream the git clone, rather than having to clone to fs on vercel, making it much faster.
# Cache and log for github
- In https://cache.forgithub.com/{owner}/{repo}/{issues|discussions|pulls}/{number} ensure to respond with a file object
- present every thread as JSON and MD
- After updating this, confirm navigating works for issues, discussions, pulls
- 🤔 Figure out how to apply plugins when there is no room for it in the URL (there is no tree) - possibly, adhere to some other standard that can also be easily generated from the frontend through a string-replace mechanism.
- Do the same for https://log.forgithub.com making these paths available. Confirm it works.
# Plugin monetisation incentivization
Come up with a good way for any plugin to provide FREEMIUM functionality. how old is the data, is the data fresh? is there a better version of the same data? if so, how can it be made available? Maybe the plugin should return one of these in response headers: Age, Date, Last-Modified, ETag, Link header (RFC 8288), Warning header (RFC 7234) - https://datatracker.ietf.org/doc/html/rfc7234#section-5.5. Maybe I could do it by just adding a README file (or similar) to the files for any ingest plugin. For a transformation plugin, a README.md or `WARNING.md` file can be added as well. This should get a special place in the UI. For example, we can render the markdown in `WARNING.md` on top, if available, so it can reach things.
TODO:
- When building the tree, also collect important files: `README.md`, `WARNING.md`, `context.json`, `.genignore`, `package.json`, and others. This is useful to determine custom frontend stuff
- Render warning if present, rendering its markdown in the UI.
# xymake zips
Fastest way to get FREE one-time package for anyone else after oauth
Fastest way to get FREE one-time package for yourself after oauth
Ensure to add monetisation URL to x router; `premiumUrl` which would allow setting scopes for regeneration.
THIS IS HOW I MAKE MONEY.
# `transform-formdata` plugins
- Should already work.
- Make this work for SWC + parsing as primary example.
- The dream: have all source-context for every codefile bound to the file (e.g. as a giant comment)
# `transform-file` plugins
Also think about the `transform-file` datatype which just passes each file that fits the mediatype to the endpoint as long as its content match the schema (if present). ActionSchema! This is tricky when doing this as a URL pipe, but can be done using `transformfile.uithub.com/{endpointurl}`. At some point we're gonna have to pass more parameters to it though, but this could be done using OTP.
Maybe also needs to be https://transformfile.uithub.com/{openapiUrl}/{operationId}
Similarly, maybe ingest plugins need openapi+operation (to know output type beforehand, to know if a plugin is another source)
# generate `.genignore` files
Can be done based on filetree and the default genignore with max certain maxtokens. Can be done using deepseek or cloudflare-based model. Can be done on-demand, stored in KV.
Fix paymentflow. ❌ Sponsorflare Sponsoring isn't working for new sponsors. Fix this by looking at changes too (or let's move to Stripe?)
Put sponsorflare in front, require signin, require balance > -1.
⏳ **Ongoing**: response and/or issue/pr for other providers to also support `.genignore` rather than just `.{platform}ignore`. 🟢 repoprompt. 🟠 repomix. 🟢 gitingest. 🟠 cursor. (RESEARCH OTHERS)
# `context.json`
Separate requests in `filter.js` that looks for `data.context` and if it exists, render questions. Add `filter.js` tab!
In uithub, magic filter that creates path filters, and fills into search and localStorage.
From filter page, if filter is applied, add button `Add to context.json` that adds copies full `context.json` or one item into clipboard, then navigates to edit/create that file (depending if it pre-existed).
If no filter is applied, add button to generate custom `context.json`
https://contextjson.com/owner/repo: Separate tool to generate a new `context.json` based on tree+README (https://uuithub.com/owner/repo?pathPatterns=README.md), and add to your project via github.
# The "agent-friendly" convention + middleware
Besides `llms.txt`, a `tree.json` file seems useful, not to put in your repo, but to be served at your website itself. In the end, it can be a convention + a middleware that serves all these things, all using the uithub API + caching based on last deployment date.
- tree.json
- llms.txt
- llms-full.txt
- .genignore
- context.json
- `.well-known/*`
- archive.zip
- archive/{contextId}.zip
Any server that exposes this sourcecode-based middleware (setting env of sha at deploy-time) is much more agent-friendly with a single LOC, and can be accessed through uuithub.com/{domain} with all search abilities there!
# `uithub.otp`
# `monetaryurl`
# Dataset
After this is there, and after I have proper categorisation, create a set of datasets with the most useful data around github repos, organised per repo.
- popular python
- popular node
- popular cloudflare
- janwilmake
- etc.
These datasets should be able to be downloaded directly as zip from some page.
# plugins i rly wnt
- monoflare to cloudflare
- cloudflare to bundled, deployable, cloudflare
- A plugin for bundling in general
- npm install
- swc parse + simplification
- typescript to typescript + bundle of entire context/dependencies
- uithub.filter
- uithub.llms
## /context.json
```json path="/context.json"
{
"$schema": "https://contextjson.com/schema",
"context": {
"example": {
"summary": "This is an example context. Change it into yours",
"pathPatterns": []
}
}
}
```
## /convention.md
# `.genignore`
`.genignore` is a proposal to standardardize how to specify which files are to be ignored for generative AI. Any implementation of genignore should follow the same spec as gitignore: https://git-scm.com/docs/gitignore
Repomix [uses](https://github.com/yamadashy/repomix/blob/main/src/core/file/fileSearch.ts) `.repomixignore`, gitingest [has considered](https://github.com/cyclotruc/gitingest/issues/147) `.gitingestignore`, repoprompt [uses](https://repoprompt.com/docs#s=file-selection&ss=filtering-files) `.repo_ignore`, cursor [uses](https://docs.cursor.com/context/ignore-files) `.cursorignore`.
uithub implements a slightly simplified version of `.genignore`: only the `.genignore` at the root is applied. I hope other context selection tools will do the same. Everyone can just add support for `.genignore` besides their own way, to give maintainers the choice to immediately support all tools.
# File filter params
UIT plugins are incentivized to stay as close to the base params as possible when possible. The following filter params are proposed to be standardized as filter parameters. any `ingest` plugin should implement them:
- `basePath:string[]` - only include files in these basePath(s)
- `pathPatterns:string[]` - glob pattern(s) of paths to be included
- `excludePathPatterns:string[]` - glob pattern(s) of paths to be excluded
- `enableFuzzyMatching` - use fuzzy matching (a la VSCode) if provided
- `omitBinary:boolean` - omit binary files if true
- `maxFileSize:number`
- `genignore:boolean` - whether or not to apply (default) `.genignore`
# `context.json`
`context.json` is a new proposed standard for managing multiple LLM contexts and providing metadata for creating AI and human interfaces to find context. [context.schema.json](https://github.com/janwilmake/uit/blob/main/uithub/static/context.schema.json)
## /process-formdata.drawio.png
Binary file available at https://raw.githubusercontent.com/janwilmake/uit/refs/heads/main/process-formdata.drawio.png
## /routers/ADR.md
# uit.com
If i have uit.com, this can become easier, as https://uithub.com can redirect to uit.com/github/... and we don't need to have the owner/repo thing.
url convention idea: https://x.com/janwilmake/status/1908066262514848040
# backtracking
The simple convention could be the following, like i also planned for zipobject:
- uit/x/dhh/lists --> https://xymake.com/dhh/lists/archive.zip or https://xymake.com/dhh/archive.zip#lists
What's useful about this is that xymake can choose to cache things at a more nested level for optimal efficiency.
# modularity of routing different domains
There's different ways to structure this:
- as independently hosted workers
- as packages
- as part of uithub
The decision on how to do this isn't easy. Ultimately, a lot of people might be making their own routers that may be dynamically added to the uithub explorer. However, this is currently far from the case. At this point I am the only maintainer, and it's just easier to invite people to make PRs than it is to ask them to make and serve indepdenent workers.
The advantage of independent worker code would be that deployment can be done independently, however, it's also a worse developer experience, and it's possible that it makes it slower (if you bind workers, the advantage of independent deployment goes away).
The advantage of packages would be that they are automatically installed/runnable when you install uithub locally, and they can be independently maintained. The disadvantage is that it adds a lot of friction for myself.
The advantage of making it just a part of uithub explorer is that it stays very simple. Location of behavior is super dense, making it easy to maintain from a single maintainer perspective.
## /routers/COVERAGE.md
## github.com (default router)
| Resource Type | URL Pattern | Routing Complete | Implementation Done | Stable |
| -------------------------------------- | ----------------------------------------------------------------- | ---------------- | ------------------- | ------ |
| User Profile, starred, repos, projects | uithub.com/{owner} | ✅ | ❌ | ❌ |
| Repository | uithub.com/{owner}/{repository} | ✅ | ✅ | ❌ |
| Repository Wiki | uithub.com/{owner}/{repository}/wiki | ✅ | 🟠 | ❌ |
| Specific Branch | uithub.com/{owner}/{repository}/tree/{branch_name} | ✅ | ✅ | ✅ |
| File or Directory | uithub.com/{owner}/{repository}/blob/{branch_name}/{path_to_file} | ✅ | ✅ | ✅ |
| Repository Issues | uithub.com/{owner}/{repository}/issues | ✅ | 🟠 | ❌ |
| Specific Issue | uithub.com/{owner}/{repository}/issues/{issue_number} | ✅ | 🟠 | ❌ |
| Pull Requests | uithub.com/{owner}/{repository}/pulls | ✅ | 🟠 | ❌ |
| Specific Pull Request | uithub.com/{owner}/{repository}/pull/{pr_number} | ✅ | 🟠 | ❌ |
| Repository Discussions | uithub.com/{owner}/{repository}/discussions | ✅ | 🟠 | ❌ |
| Specific Discussions | uithub.com/{owner}/{repository}/discussions/{discussion_number} | ✅ | 🟠 | ❌ |
| Repository Branches | uithub.com/{owner}/{repository}/branches | ✅ | ❌ | ❌ |
| Repository Commits | uithub.com/{owner}/{repository}/commits | ✅ | ❌ | ❌ |
| Specific Commit | uithub.com/{owner}/{repository}/commit/{commit_hash} | ✅ | ❌ | ❌ |
| Repository Releases | uithub.com/{owner}/{repository}/releases | ✅ | ❌ | ❌ |
| Specific Release | uithub.com/{owner}/{repository}/releases/tag/{tag_name} | ✅ | ❌ | ❌ |
| Repository Actions | uithub.com/{owner}/{repository}/actions | ✅ | ❌ | ❌ |
| Compare Changes | uithub.com/{owner}/{repository}/compare/{base}...{head} | ✅ | ❌ | ❌ |
| Specific Star List | uithub.com/stars/{owner}/lists/{list_id} | ✅ | ❌ | ❌ |
| Organization | uithub.com/orgs/{org_name} | ✅ | ❌ | ❌ |
| Starred Repositories | uithub.com/stars/{owner} | ❌ | ❌ | ❌ |
| Repository Projects | uithub.com/{owner}/{repository}/projects | ❌ | ❌ | ❌ |
| Gists | uithub.com/{owner}/gists/{gist_id} | ❌ | ❌ | ❌ |
## npmjs.com
| Resource Type | URL Pattern | Routing Complete | Implementation Done | Stable |
| ---------------------- | ---------------------------------------------------------------- | ---------------- | ------------------- | ------ |
| Base route | uithub.com/npmjs.com/ | ❌ | ❌ | ❌ |
| Package (regular) | uithub.com/npmjs.com/package/[package_name] | ✅ | ✅ | ✅ |
| Package (scoped) | uithub.com/npmjs.com/package/@[scope]/[package_name] | ✅ | ✅ | ✅ |
| Package version | uithub.com/npmjs.com/package/[package_name]/v/[version] | ✅ | ✅ | ✅ |
| Scoped package version | uithub.com/npmjs.com/package/@[scope]/[package_name]/v/[version] | ✅ | ✅ | ✅ |
## x.com
| Resource Type | URL Pattern | Routing Complete | Implementation Done | Stable |
| ----------------------------- | ---------------------------------------- | ---------------- | ------------------- | ------ |
| User (all username endpoints) | uithub.com/x.com/{username} | ✅ | ❌ | ❌ |
| List details and members | uithub.com/x.com/i/lists/[list_id] | ✅ | ❌ | ❌ |
| Bookmarks | uithub.com/x.com/i/bookmarks | ❌ | ❌ | ❌ |
| Topics | uithub.com/x.com/i/topics | ❌ | ❌ | ❌ |
| Spaces | uithub.com/x.com/i/spaces | ❌ | ❌ | ❌ |
| Communities | uithub.com/x.com/i/communities | ❌ | ❌ | ❌ |
| Home timeline | uithub.com/x.com/home | ❌ | ❌ | ❌ |
| Messages | uithub.com/x.com/messages | ❌ | ❌ | ❌ |
| Notifications | uithub.com/x.com/notifications | ❌ | ❌ | ❌ |
| Explore | uithub.com/x.com/explore | ❌ | ❌ | ❌ |
| Search | uithub.com/x.com/search?q=[search_terms] | ❌ | ❌ | ❌ |
# openapisearch.com
OpenAPISearch will provide crawled openapis in converted formats as a agent-supervised dataset. This is high prio and will be coming soon. Reach out if you have specific needs to an openapi dataset.
Here we wanna find a url structure as well that works nicely for domains as well as for github repos. the simplest way is probably:
- decode url
- remove protocol `https?://` (redirect)
- remove `.com` (redirect)
- remove suffix `/openapi.json` (redirect)
- keep a harcoded list of storage domains such as github, npm, etc, that allow alternative name:
- openapisearch.com/https://github.com/janwilmake/openapisearch/tree/main/openapi.json -> openapisearch/janwilmake_openapisearch.githus
- openapisearch/janwilmake_openapisearch.githus/handmade/brandwatch.json
## /routers/README.md
# UIT routing convention (RFC draft)
The goal for uithub is to support the entire url structure of major websites, and make its data accessible to APIs and LLMs at high scale. uithub brings the UIT protocol to the browser by routing any URL structure to the right source, plugin, and query params. This allows uithub to view context for any domain.
uithub's routing applies the following rules:
1. `github` is the default applied router
2. if the first segment is of format `{domain.tld}`, another router gets applied to retrieve the source
3. if there is no router for `{domain.tld}` the `domain crawling convention` is applied.
# Conventions in the URL Structure
A regular uithub URL pathname consists of these parts, all are optional:
- `domain`
- `pluginId`
- extension (determines desired output format)
- source locator (can consist of `primarySourceSegment` and `secondarySourceSegment`)
- `basePath`
This allow accessing any source and explore/transform it in different ways.
| Feature | URL Pattern | Description | Routing Complete | Implementation Done | Stable |
| ---------------------- | -------------------------------------------------------------- | ---------------------------------------------------------- | ---------------- | ------------------- | ------ |
| Domain specific router | uithub.com/**{domain.tld}** | Use domain as owner with repository being an ID | ✅ | ✅ | ✅ |
| Alternative Page Types | uithub.com/{owner}/{repository}/**{pluginId}**/{branch}/{path} | Page type can be any compatible plugin | ✅ | ✅ | ❌ |
| Extended Format | uithub.com/{owner}/{repository}/**{pluginId}.{ext}** | Add file extension to page type for different data formats | ✅ | ✅ | ❌ |
The following is an example for GitHub, which is also the standard router used when no domain is specified:
# Domain crawling convention
The uit routing system applies the following rules when a domain does not have its own specified router API. This convention is experimental and will have breaking changes. Feedback much appreciated, let's come up with the best convention together!
1. look for `/archive.zip` at the domain root. If found, this is used as source. We can also conventionalise "backracking" support, which would search for archive.zip for any path and go back he pathname until found.
2. use the `reader.llmtext.com/{domain.tld}/{basePath}` router if not found, which will crawl the website (WIP)
# Creating a router
Anyone can create a website-router to become viewable by uithub. For any given domain, uithub will adopt community-contributed routers of popular websites until the website owner provides their own official router.
Your domain specific router should mirror the `GET` URL structure of the original domain, map the `pathname` to a StandardURL JSON Response. See [standard-url.d.ts](./standard-url.d.ts) for the most up-to-date specification.
# Implemented routers (so far)
See [COVERAGE.md](COVERAGE.md)
## /routers/StandardURL.ts
```ts path="/routers/StandardURL.ts"
export type StandardURL = {
/**
* If given, will be used to ingest the data
*
* If not given, the ingestUrl will be the same route (with accept application/form-data) */
sourceUrl: string;
sourceType: "zip" | "json" | "tar" | "formdata";
// This is all to render the frontend
omitFirstSegment?: boolean;
primarySourceSegment: string;
pluginId?: string;
secondarySourceSegment?: string;
basePath?: string;
ext?: string;
ogImageUrl?: string;
title?: string;
description?: string;
rawUrlPrefix?: string;
baseLink: string;
moreToolsLink?: string;
/** Optional: a record map of basePaths (without '/' prefix) and the values being menu titles */
navigation?: Record;
};
```
## /routers/github.ts
```ts path="/routers/github.ts"
import { StandardURL } from "./StandardURL";
// Helper function to get zip URL for a repo
function getRepoZipUrl(
owner: string,
repo: string,
branch: string | undefined,
isAuthenticated: boolean,
): string {
const ref = /^[0-9a-f]{40}$/i.test(branch || "")
? branch
: `refs/heads/${branch || "main"}`;
return isAuthenticated
? `https://api.github.com/repos/${owner}/${repo}/zipball${
branch ? "/" + branch : ""
}`
: `https://github.com/${owner}/${repo}/archive/${ref}.zip`;
}
export default {
fetch: async (request: Request): Promise => {
const pathname = new URL(request.url).pathname;
const isAuthenticated =
request.headers.get("X-IS-AUTHENTICATED") === "true";
const baseLink = `https://github.com${pathname}`;
const moreToolsLink = `https://forgithub.com${pathname}`;
let navigation = { "": "Home" };
// Parse GitHub path components
let [owner, repo, pageAndExt, branch, ...pathParts] = pathname
.split("/")
.slice(1);
const [page, ext] = (pageAndExt || "").split(".");
if (!owner || owner === "" || owner === "-") {
const [pluginIdAndExt, ...basePathParts] = pathname.split("/").slice(2);
const [pluginId, ext] = (pluginIdAndExt || "").split(".");
let title = `Popular repos`;
let description = `Popular repos on GitHub`;
const json: StandardURL = {
navigation,
baseLink,
moreToolsLink,
pluginId,
ext,
basePath: basePathParts.join("/"),
primarySourceSegment: `-`,
title,
description,
omitFirstSegment: false,
sourceUrl: `https://popular.forgithub.com/index.json`,
sourceType: "json",
};
return new Response(JSON.stringify(json, undefined, 2), {
headers: { "content-type": "application/json" },
});
}
// Handle organization pages (/orgs/{orgName}/{subpage}/{pluginIdAndExt}/{...basePath})
if (owner === "orgs" && repo) {
const [orgName, subpage, pluginIdAndExt, ...basePathParts] = pathname
.split("/")
.slice(2);
const [pluginId, ext] = (pluginIdAndExt || "").split(".");
let sourceUrl = `https://cache.forgithub.com/orgs/${orgName}`;
let title = `GitHub Organization ${orgName}`;
let description = `Information about the ${orgName} organization`;
const basePath = subpage ? [subpage].concat(basePathParts).join("/") : "";
const json: StandardURL = {
navigation,
baseLink,
moreToolsLink,
pluginId,
ext,
basePath,
primarySourceSegment: `orgs/${orgName}${subpage ? "/" + subpage : ""}`,
secondarySourceSegment: basePathParts.join("/"),
title,
description,
sourceType: "json",
omitFirstSegment: false,
sourceUrl,
};
return new Response(JSON.stringify(json, undefined, 2), {
headers: { "content-type": "application/json" },
});
}
// Handle user profile
// ex. /{owner}/-/{pluginId}/{...basePath}
if (!repo || repo === "-") {
const [owner, _, pluginIdAndExt, ...basePathParts] = pathname
.split("/")
.slice(1);
const basePath = basePathParts.join("/");
const [pluginId, ext] = (pluginIdAndExt || "").split(".");
const json: StandardURL = {
navigation,
baseLink,
moreToolsLink,
// no branch
basePath,
primarySourceSegment: owner + "/-",
pluginId,
ext,
title: `GitHub User ${owner}`,
description: `Profile information, repositories, stars, and projects for ${owner}`,
sourceType: "json",
omitFirstSegment: false,
sourceUrl: `https://cache.forgithub.com/stars/${owner}`,
};
return new Response(JSON.stringify(json, undefined, 2), {
headers: { "content-type": "application/json" },
});
}
// Handle GitHub lists structure
if (owner === "stars" && page === "lists" && branch) {
const listName = branch;
owner = repo;
const [pageAndExt, branchName, ...basePathParts] = pathParts;
const basePath = basePathParts.join("/");
const [pluginId, ext] = (pageAndExt || "").split(".");
const zipUrl = `https://lists.forgithub.com/${owner}/${listName}${
branchName ? `/${branchName}.zip` : ".zip"
}`;
const primarySourceSegment = pathname.split("/").slice(1, 4).join("/");
const json: StandardURL = {
navigation,
baseLink,
moreToolsLink,
pluginId,
ext,
basePath,
primarySourceSegment,
secondarySourceSegment: branchName || "main",
description: `Easily ask your LLM code questions about "${listName}". /${basePath} on GitHub.`,
title: `GitHub list ${listName} LLM Context`,
sourceType: "zip",
omitFirstSegment: false,
sourceUrl: zipUrl,
};
return new Response(JSON.stringify(json, undefined, 2), {
headers: { "content-type": "application/json" },
});
}
if (owner === "stars" && repo) {
// redirect to the user page
return new Response("{}", {
status: 307,
headers: { Location: `/${repo}/stars` },
});
}
const basePath = pathParts.join("/");
const primarySourceSegment = `${owner}/${repo}`;
navigation = {
...navigation,
[`${owner}/${repo}`]: "Code",
[`${owner}/${repo}/issues`]: "Issues",
[`${owner}/${repo}/pulls`]: "Pull Requests",
[`${owner}/${repo}/discussions`]: "Discussions",
[`${owner}/${repo}/branches`]: "Branches",
[`${owner}/${repo}/commits`]: "Commits",
[`${owner}/${repo}/releases`]: "Releases",
};
// Handle GitHub source types
switch (page) {
case "wiki": {
const json: StandardURL = {
navigation,
baseLink,
moreToolsLink,
pluginId: branch || "tree",
ext,
basePath,
primarySourceSegment: `${owner}/${repo}/wiki`,
title: `GitHub ${primarySourceSegment} Wiki`,
description: `Wiki documentation for ${primarySourceSegment}`,
sourceType: "zip",
omitFirstSegment: false,
sourceUrl: `https://wikizip.forgithub.com/${primarySourceSegment}`,
};
return new Response(JSON.stringify(json, undefined, 2), {
headers: { "content-type": "application/json" },
});
}
case "compare": {
// For compare, we expect format: /owner/repo/compare/base...head
const compareParams = branch; // branch holds "base...head"
if (compareParams && compareParams.includes("...")) {
const [base, head] = compareParams.split("...");
// Use the new function to get zip URLs for both base and head
const baseZipUrl = encodeURIComponent(
getRepoZipUrl(owner, repo, base, isAuthenticated),
);
const headZipUrl = encodeURIComponent(
getRepoZipUrl(owner, repo, head, isAuthenticated),
);
const json: StandardURL = {
navigation,
baseLink,
moreToolsLink,
pluginId: page,
ext,
basePath,
primarySourceSegment,
secondarySourceSegment: compareParams,
title: `Compare ${base}...${head} in ${primarySourceSegment}`,
description: `Compare changes between ${base} and ${head} branches`,
sourceType: "zip",
omitFirstSegment: false,
sourceUrl: `https://compare.uithub.com/${baseZipUrl}/${headZipUrl}`,
};
return new Response(JSON.stringify(json, undefined, 2), {
headers: { "content-type": "application/json" },
});
}
break;
}
case "issues": {
const json: StandardURL = {
navigation,
baseLink,
moreToolsLink,
pluginId: page,
ext,
basePath,
primarySourceSegment,
secondarySourceSegment: "issues",
title: `GitHub ${primarySourceSegment} Issues`,
description: `LLM context for issues in ${primarySourceSegment}`,
sourceType: "json",
omitFirstSegment: false,
sourceUrl: `https://cache.forgithub.com/${primarySourceSegment}/issues`,
};
return new Response(JSON.stringify(json, undefined, 2), {
headers: { "content-type": "application/json" },
});
}
case "pull":
case "pulls": {
const json: StandardURL = {
navigation,
baseLink,
moreToolsLink,
pluginId: page,
ext,
basePath,
primarySourceSegment,
secondarySourceSegment: "pulls",
title: `GitHub ${primarySourceSegment} Pull Requests`,
description: `LLM context for pull requests in ${primarySourceSegment}`,
sourceType: "json",
omitFirstSegment: false,
sourceUrl: `https://cache.forgithub.com/${primarySourceSegment}/pulls`,
};
return new Response(JSON.stringify(json, undefined, 2), {
headers: { "content-type": "application/json" },
});
}
case "discussions": {
const json: StandardURL = {
navigation,
baseLink,
moreToolsLink,
pluginId: page,
ext,
basePath,
primarySourceSegment,
secondarySourceSegment: "discussions",
title: `GitHub ${primarySourceSegment} Discussions`,
description: `LLM context for discussions in ${primarySourceSegment}`,
sourceType: "json",
omitFirstSegment: false,
sourceUrl: `https://cache.forgithub.com/${primarySourceSegment}/discussions`,
};
return new Response(JSON.stringify(json, undefined, 2), {
headers: { "content-type": "application/json" },
});
}
case "branches": {
const json: StandardURL = {
baseLink,
moreToolsLink,
navigation,
pluginId: page,
ext,
basePath,
primarySourceSegment,
secondarySourceSegment: "branches",
title: `GitHub ${primarySourceSegment} Branches`,
description: `Branch information and last commits for ${primarySourceSegment}`,
sourceType: "json",
omitFirstSegment: false,
sourceUrl: `https://log.forgithub.com/${primarySourceSegment}/branches`,
};
return new Response(JSON.stringify(json, undefined, 2), {
headers: { "content-type": "application/json" },
});
}
case "commits":
case "commit": {
const json: StandardURL = {
navigation,
baseLink,
moreToolsLink,
pluginId: page,
ext,
basePath,
primarySourceSegment,
secondarySourceSegment: "commits",
title: `GitHub ${primarySourceSegment} Commits`,
description: `Commit history and contributor info for ${primarySourceSegment}`,
sourceType: "json",
omitFirstSegment: false,
sourceUrl: `https://log.forgithub.com/${primarySourceSegment}/commits`,
};
return new Response(JSON.stringify(json, undefined, 2), {
headers: { "content-type": "application/json" },
});
}
case "releases": {
const json: StandardURL = {
navigation,
baseLink,
moreToolsLink,
pluginId: page,
ext,
basePath,
primarySourceSegment,
secondarySourceSegment: "releases",
title: `GitHub ${primarySourceSegment} Releases`,
description: `Release information for ${primarySourceSegment}`,
sourceType: "json",
omitFirstSegment: false,
sourceUrl: `https://log.forgithub.com/${primarySourceSegment}/releases`,
};
return new Response(JSON.stringify(json, undefined, 2), {
headers: { "content-type": "application/json" },
});
}
case "actions": {
const json: StandardURL = {
navigation,
baseLink,
moreToolsLink,
pluginId: page,
ext,
basePath,
primarySourceSegment,
secondarySourceSegment: "actions",
title: `GitHub ${primarySourceSegment} Actions`,
description: `GitHub Actions workflows for ${primarySourceSegment}`,
sourceType: "json",
omitFirstSegment: false,
sourceUrl: `https://actions.forgithub.com/${primarySourceSegment}`,
};
return new Response(JSON.stringify(json, undefined, 2), {
headers: { "content-type": "application/json" },
});
}
}
// Default repository/file handling
const zipUrl = getRepoZipUrl(owner, repo, branch, isAuthenticated);
const currentTokens = `{{currentTokens}}`;
const ogImageUrl = `https://github-og-image.githuq.workers.dev/${primarySourceSegment}?path=${basePath}&tokens=${currentTokens}`;
const ref = /^[0-9a-f]{40}$/i.test(branch || "")
? branch
: `refs/heads/${branch || "main"}`;
const rawUrlPrefix = `https://raw.githubusercontent.com/${owner}/${repo}/${ref}`;
const json: StandardURL = {
navigation,
baseLink,
moreToolsLink,
pluginId: page,
ext,
basePath,
primarySourceSegment,
secondarySourceSegment: branch || "main",
ogImageUrl,
description: `Easily ask your LLM code questions about "${primarySourceSegment}". /${basePath} on GitHub contains ${currentTokens} tokens.`,
title: `GitHub ${primarySourceSegment} LLM Context`,
sourceType: "zip",
omitFirstSegment: true,
sourceUrl: zipUrl,
rawUrlPrefix,
};
return new Response(JSON.stringify(json, undefined, 2), {
headers: { "content-type": "application/json" },
});
},
};
```
## /routers/npmjs.ts
```ts path="/routers/npmjs.ts"
import { StandardURL } from "./StandardURL";
/**
* This is the domain-specific router that determines the domain-specific decomposition
* of the pathname and the source URL for npmjs.com
*/
export default {
fetch: async (request: Request): Promise => {
const pathname = new URL(request.url).pathname;
const isAuthenticated =
request.headers.get("X-IS-AUTHENTICATED") === "true";
// Check if the URL starts with /package/
if (!pathname.startsWith("/package/")) {
return new Response("Invalid npm package URL format", { status: 404 });
}
// Get the package path after /package/
const remainingPath = pathname.substring("/package/".length);
const segments = remainingPath.split("/").filter((part) => part);
// State variables
let packageName = "";
let scope = "";
let version = "";
let pluginId = "";
let ext = "";
let basePath = "";
// Parse package name (with optional scope)
if (segments[0]?.startsWith("@")) {
// Scoped package: @scope/package
if (segments.length < 2) {
return new Response("Invalid scoped package format", { status: 404 });
}
scope = segments[0].substring(1); // Remove @
packageName = segments[1];
segments.splice(0, 2); // Remove scope and package name
} else {
// Regular package
if (segments.length === 0) {
return new Response("Invalid package format", { status: 404 });
}
packageName = segments[0];
segments.splice(0, 1); // Remove package name
}
// Check for optional /v/{version} format
if (
segments.length >= 2 &&
segments[0] === "v" &&
segments[1].match(/^\d/)
) {
version = segments[1];
segments.splice(0, 2); // Remove 'v' and version
}
// Extract plugin information if present (first segment with a dot)
if (segments.length > 0) {
const [id, extension] = segments[0].split(".");
pluginId = id;
ext = extension;
segments.splice(0, 1); // Remove plugin segment
}
// Remaining segments form the basePath
basePath = segments.join("/");
// If no version was found in URL, fetch latest version
if (!version) {
try {
const registryUrl = scope
? `https://registry.npmjs.org/@${scope}/${packageName}/latest`
: `https://registry.npmjs.org/${packageName}/latest`;
const response = await fetch(registryUrl);
const json: { version: string } = await response.json();
version = json.version;
if (!version) {
throw new Error("No version found");
}
} catch (e) {
return new Response("Failed to fetch package version", { status: 404 });
}
}
// Construct full package name and tarball URL
const fullPackageName = scope ? `@${scope}/${packageName}` : packageName;
const tarballUrl = `https://registry.npmjs.org/${fullPackageName}/-/${packageName}-${version}.tgz`;
// Generate metadata
const currentTokens = `{{currentTokens}}`;
const title = `NPM ${fullPackageName} LLM Context`;
const description = `Easily ask your LLM code questions about "${fullPackageName}" npm package. /${basePath} contains ${currentTokens} tokens.`;
const ogImageUrl = `https://github-og-image.githuq.workers.dev/npm/${fullPackageName}?path=${basePath}&tokens=${currentTokens}`;
const rawUrlPrefix = `https://unpkg.com/${fullPackageName}@${version}`;
// Create the StandardURL response
const json: StandardURL = {
pluginId,
ext,
basePath,
primarySourceSegment: "package/" + fullPackageName + "/v/" + version,
ogImageUrl,
description,
title,
omitFirstSegment: true,
sourceType: "tar",
baseLink: "https://npmjs.com" + pathname,
sourceUrl: tarballUrl,
rawUrlPrefix,
};
return new Response(JSON.stringify(json, undefined, 2), {
headers: { "content-type": "application/json" },
});
},
};
```
## /routers/router-simple.drawio.png
Binary file available at https://raw.githubusercontent.com/janwilmake/uit/refs/heads/main/routers/router-simple.drawio.png
## /routers/router.drawio.png
Binary file available at https://raw.githubusercontent.com/janwilmake/uit/refs/heads/main/routers/router.drawio.png
## /routers/x.ts
```ts path="/routers/x.ts"
// prompt
// https://raw.githubusercontent.com/janwilmake/xymake/refs/heads/main/README.md
// + github.ts
// Similar to the GitHub router, please create a x.ts router that returns a StandardURL for every endpoint.all username endpoints can be grouped into a single one. all locked ones need not be done yet (return 404 for these, and others). so basically we only have i/lists/{id} and username/{page} and username/status/{id}. behind each we need to parse {pluginId} and after that {pathParts}.
// what needs to be specified, is the sources that are available.
// TODO
// 1. check the url structure on X for username. is it really just the username/status/{id} and username/{page} or is there more to it?
// 2. if its really just that, either decide to have the plugin BEFORE the username, or after the username.
import { StandardURL } from "../StandardURL";
export default {
fetch: async (request: Request): Promise => {
const pathname = new URL(request.url).pathname;
const isAuthenticated =
request.headers.get("X-IS-AUTHENTICATED") === "true";
// Parse path components
const [firstSegment, secondSegment, ...restParts] = pathname
.split("/")
.slice(1);
// Handle list details: /i/lists/{list_id}/{pluginIdAndExt}/{...basePath}
if (firstSegment === "i" && secondSegment === "lists") {
const [listId, pluginIdAndExt, ...basePathParts] = restParts;
if (!listId) {
return new Response("List ID required", { status: 400 });
}
const [pluginId, ext] = (pluginIdAndExt || "").split(".");
const json: StandardURL = {
pluginId,
ext,
basePath: basePathParts.join("/"),
primarySourceSegment: `i/lists/${listId}`,
title: `List ${listId}`,
description: `List details and members for list ${listId}`,
sourceType: "json",
omitFirstSegment: false,
sourceUrl: `https://api.xymake.com/i/lists/${listId}.json`,
};
return new Response(JSON.stringify(json, undefined, 2), {
headers: { "content-type": "application/json" },
});
}
// Handle username-based endpoints
if (firstSegment) {
const username = firstSegment;
// /[username]/status/[tweet_id]/{pluginIdAndExt}/{...basePath}
if (secondSegment === "status") {
const [tweetId, pluginIdAndExt, ...basePathParts] = restParts;
if (!tweetId) {
return new Response("Tweet ID required", { status: 400 });
}
const [pluginId, ext] = (pluginIdAndExt || "").split(".");
const json: StandardURL = {
pluginId,
ext,
basePath: basePathParts.join("/"),
primarySourceSegment: `${username}/status/${tweetId}`,
title: `Tweet by @${username}`,
description: `Tweet ${tweetId} by @${username} and its replies`,
sourceType: "json",
omitFirstSegment: false,
sourceUrl: `https://xymake.com/${username}/status/${tweetId}.json`,
};
return new Response(JSON.stringify(json, undefined, 2), {
headers: { "content-type": "application/json" },
});
}
// Handle locked/unimplemented endpoints
const lockedEndpoints = [
"following",
"followers",
"verified_followers",
"creator-subscriptions",
"photo",
"articles",
"media",
"likes",
"bookmarks",
];
if (lockedEndpoints.includes(secondSegment)) {
return new Response(
JSON.stringify({
error: "Endpoint requires X API Basic Plan",
message: `The ${secondSegment} endpoint is not available in the free tier`,
}),
{
status: 403,
headers: { "content-type": "application/json" },
},
);
}
// Handle standard user endpoints: /[username]/{page}/{pluginIdAndExt}/{...basePath}
let page = secondSegment;
let pluginIdAndExt;
let basePathParts;
if (page) {
[pluginIdAndExt, ...basePathParts] = restParts;
} else {
// Handle /[username]/{pluginIdAndExt}/{...basePath}
page = "profile";
[pluginIdAndExt, ...basePathParts] = [secondSegment, ...restParts];
}
const [pluginId, ext] = (pluginIdAndExt || "").split(".");
let title = `@${username}`;
let description = `X profile for @${username}`;
let sourceUrl = `https://api.xymake.com/${username}.json`;
switch (page) {
case "with_replies":
title = `@${username} with replies`;
description = `Tweets and replies from @${username}`;
sourceUrl = `https://api.xymake.com/${username}/with_replies.json`;
break;
case "highlights":
title = `@${username} highlights`;
description = `Highlighted tweets from @${username}`;
sourceUrl = `https://api.xymake.com/${username}/highlights.json`;
break;
case "lists":
title = `Lists by @${username}`;
description = `Lists created or subscribed to by @${username}`;
sourceUrl = `https://api.xymake.com/${username}/lists.json`;
break;
}
const json: StandardURL = {
pluginId,
ext,
basePath: basePathParts?.join("/") || "",
primarySourceSegment: username,
secondarySourceSegment: page !== "profile" ? page : undefined,
title,
description,
sourceType: "json",
omitFirstSegment: false,
sourceUrl,
};
return new Response(JSON.stringify(json, undefined, 2), {
headers: { "content-type": "application/json" },
});
}
// Handle other endpoints (explore, search, etc.) - return 404 for now
const unimplementedEndpoints = [
"home",
"messages",
"notifications",
"explore",
"search",
"i/bookmarks",
"i/topics",
"i/spaces",
"i/communities",
];
if (
unimplementedEndpoints.includes(firstSegment) ||
(firstSegment === "i" &&
unimplementedEndpoints.includes(`i/${secondSegment}`))
) {
return new Response(
JSON.stringify({
error: "Not Implemented",
message: "This endpoint has not been implemented yet",
}),
{
status: 404,
headers: { "content-type": "application/json" },
},
);
}
// Default 404 response
return new Response(
JSON.stringify({
error: "Not Found",
message: "The requested endpoint does not exist",
}),
{
status: 404,
headers: { "content-type": "application/json" },
},
);
},
};
```
## /routers/ycombinator.news.ts
```ts path="/routers/ycombinator.news.ts"
import { StandardURL } from "./StandardURL.js";
export default {
fetch: async (request: Request): Promise => {
const url = new URL(request.url);
const pathname = url.pathname;
const searchParams = url.searchParams;
console.log({ pathname });
// Split path into segments
const pathSegments = pathname.split("/").filter(Boolean);
const baseLink = "https://news.ycombinator.com" + pathname;
// Check if this is a plugin-prefixed path or a direct HN path
let pluginIdAndExt = "tree";
let ext: string | undefined;
let hnPathSegments: string[] = [...pathSegments];
let basePath: string | undefined;
// Known Hacker News routes that should be preserved as routes
const knownHnRoutes = [
"news",
"newest",
"front",
"ask",
"show",
"jobs",
"submit",
"user",
"item",
"newcomments",
"threads",
];
// If the first segment isn't a known HN route, treat it as a pluginId
if (pathSegments.length > 0 && !knownHnRoutes.includes(pathSegments[0])) {
[pluginIdAndExt, ...hnPathSegments] = pathSegments;
const parts = pluginIdAndExt.split(".");
if (parts.length > 1) {
[pluginIdAndExt, ext] = parts;
}
}
// Parse query parameters for item?id= pattern
let secondarySourceSegment = "";
let sqlQuery = "";
let title = "Hacker News";
let description = "Hacker News content and discussions";
// Handle different HN routes
if (
hnPathSegments.length === 0 ||
hnPathSegments[0] === "news" ||
hnPathSegments[0] === "front"
) {
// Front page
secondarySourceSegment = "front";
title = "Hacker News Front Page";
description = "Top stories from Hacker News";
sqlQuery = encodeURIComponent(
`SELECT * FROM items WHERE type = 'story' AND NOT deleted AND NOT dead ORDER BY score DESC LIMIT 30`,
);
} else if (hnPathSegments[0] === "newest") {
// Newest stories
secondarySourceSegment = "newest";
title = "Newest Submissions";
description = "Most recent submissions to Hacker News";
sqlQuery = encodeURIComponent(
`SELECT * FROM items WHERE type = 'story' AND NOT deleted AND NOT dead ORDER BY time DESC LIMIT 30`,
);
} else if (hnPathSegments[0] === "ask") {
// Ask HN
secondarySourceSegment = "ask";
title = "Ask Hacker News";
description = "Questions and discussions from the Hacker News community";
sqlQuery = encodeURIComponent(
`SELECT * FROM items WHERE title LIKE 'Ask HN%' AND type = 'story' AND NOT deleted AND NOT dead ORDER BY score DESC LIMIT 30`,
);
} else if (hnPathSegments[0] === "show") {
// Show HN
secondarySourceSegment = "show";
title = "Show Hacker News";
description = "Projects and creations from the Hacker News community";
sqlQuery = encodeURIComponent(
`SELECT * FROM items WHERE title LIKE 'Show HN%' AND type = 'story' AND NOT deleted AND NOT dead ORDER BY score DESC LIMIT 30`,
);
} else if (hnPathSegments[0] === "jobs") {
// Jobs
secondarySourceSegment = "jobs";
title = "Hacker News Jobs";
description = "Job listings posted to Hacker News";
sqlQuery = encodeURIComponent(
`SELECT * FROM items WHERE type = 'job' AND NOT deleted AND NOT dead ORDER BY time DESC LIMIT 30`,
);
} else if (hnPathSegments[0] === "user") {
const username =
hnPathSegments.length > 1
? hnPathSegments[1]
: searchParams.has("id")
? searchParams.get("id")
: //default
"pg";
secondarySourceSegment = `user/${username}`;
title = `Hacker News User: ${username}`;
description = `Profile and submissions by ${username}`;
sqlQuery = encodeURIComponent(
`SELECT * FROM items WHERE by = '${username}' AND NOT deleted AND NOT dead ORDER BY time DESC LIMIT 30`,
);
} else if (hnPathSegments[0] === "item") {
// Item details - from either path or query parameter
let itemId: string | null = null;
// Check if id is in the path segments
if (hnPathSegments.length > 1) {
itemId = hnPathSegments[1];
}
// Otherwise check query parameter
else if (searchParams.has("id")) {
itemId = searchParams.get("id");
}
if (itemId && /^\d+$/.test(itemId)) {
secondarySourceSegment = `item/${itemId}`;
title = `Hacker News Item #${itemId}`;
description = `Discussion and details for item #${itemId}`;
sqlQuery = encodeURIComponent(
`SELECT * FROM items WHERE id = ${itemId}`,
);
} else {
return new Response(
JSON.stringify({
error: "Invalid Item ID",
message: "The item ID is missing or invalid",
}),
{
status: 400,
headers: { "content-type": "application/json" },
},
);
}
} else if (hnPathSegments[0] === "newcomments") {
// New comments
secondarySourceSegment = "newcomments";
title = "New Comments";
description = "Recent comments on Hacker News";
sqlQuery = encodeURIComponent(
`SELECT * FROM items WHERE has_comments = TRUE AND NOT deleted AND NOT dead ORDER BY time DESC LIMIT 30`,
);
} else if (hnPathSegments[0] === "threads" && hnPathSegments.length > 1) {
// User threads
const username = hnPathSegments[1];
secondarySourceSegment = `threads/${username}`;
title = `${username}'s Threads`;
description = `Comment threads by ${username}`;
sqlQuery = encodeURIComponent(
`SELECT * FROM items WHERE by = '${username}' AND has_comments = TRUE AND NOT deleted AND NOT dead ORDER BY time DESC LIMIT 30`,
);
} else {
// If we get here, we might have a custom path or an unsupported route
// Set the segments after the potential plugin as the basePath
basePath = hnPathSegments.join("/");
secondarySourceSegment = "custom";
title = "Hacker News Custom View";
description = "Custom view of Hacker News content";
sqlQuery = encodeURIComponent(
`SELECT * FROM items WHERE NOT deleted AND NOT dead ORDER BY time DESC LIMIT 30`,
);
}
// Handle any additional query parameters from the original request
// Construct the source URL with the DORM spec
const sourceUrl = `https://ingestsql.uithub.com/https://crawler.gcombinator.com/api/db/query/raw/${sqlQuery}?itemTemplate={id}.json`;
// Create the StandardURL response
const json: StandardURL = {
baseLink,
pluginId: pluginIdAndExt,
ext,
basePath,
primarySourceSegment: "",
secondarySourceSegment,
title,
description,
sourceType: "formdata",
omitFirstSegment: false,
sourceUrl,
};
return new Response(JSON.stringify(json, undefined, 2), {
headers: { "content-type": "application/json" },
});
},
};
```
## /uithub.ingestjson/.gitignore
```gitignore path="/uithub.ingestjson/.gitignore"
node_modules
.wrangler
package-lock.json
.dev.vars
.env
```
## /uithub.ingestjson/README.md
https://uuithub.com/janwilmake/uit/tree/main/uithub.ingestzip
just like this, make a lib called ingestjson that gets passed a json url, fetches the json into memory, then turns it into files by walking over it in this way:
if the shape of the json is { files: {[path:string]: {type:"binary"|"content", content?:string, url?:string} }, the paths are determined like that, the content comes from either content or url depending on type.
if not, the values in the first level of the json (must be object or array) will be the files. the filename is either the slug, id, or the index of the value, appended with .json
after determining the paths, the path filters are applied (in the same way as ingest zip). the output is a FormData stream with x-error and x-filter in the same way as ingestjson (x-filter if filtered out)
## /uithub.ingestjson/globals.d.ts
```ts path="/uithub.ingestjson/globals.d.ts"
declare module "*.md" {
const content: string;
export default content;
}
declare module "*.html" {
const content: string;
export default content;
}
```
## /uithub.ingestjson/main.ts
```ts path="/uithub.ingestjson/main.ts"
// Import picomatch for glob pattern matching
import picomatch from "picomatch";
import map from "./public/ext-to-mime.json";
import binaryExtensions from "binary-extensions";
import { Env, FilterOptions, RequestParams, ResponseOptions } from "./types";
export default {
async fetch(request: Request, env: Env, ctx: any): Promise {
// Parse the request parameters
const requestStartTime = Date.now();
const params = parseRequest(request);
const { jsonUrl, filterOptions, responseOptions } = params;
// Include timing info in response headers
const responseHeaders = new Headers({
"Content-Type": responseOptions.isBrowser
? `text/plain; boundary=${responseOptions.boundary}; charset=utf-8`
: `multipart/form-data; boundary=${responseOptions.boundary}`,
"Transfer-Encoding": "chunked",
});
// Validate the JSON URL
if (!jsonUrl) {
return new Response("No JSON URL provided", { status: 400 });
}
// Check authentication
if (!isAuthenticated(request, env.CREDENTIALS)) {
return new Response("Authentication required", {
status: 401,
headers: {
"WWW-Authenticate": 'Basic realm="JSON Access"',
},
});
}
try {
// Prepare headers for fetching the JSON
const headers = new Headers({ "User-Agent": "Cloudflare-Worker" });
if (responseOptions.authHeader) {
headers.set("Authorization", responseOptions.authHeader);
}
// Fetch the JSON
const jsonResponse = await fetch(jsonUrl, { headers });
if (!jsonResponse.ok) {
return createErrorResponse(jsonResponse, params.jsonUrl);
}
const initialResponseTime = Date.now() - requestStartTime;
responseHeaders.set(
"X-Initial-Response-Time-Ms",
initialResponseTime.toString(),
);
// Parse the JSON data
const jsonData = await jsonResponse.json();
// Process and stream the JSON as files
const { readable, writable } = new TransformStream();
// Start processing the JSON in the background
processJsonToMultipart(
jsonData,
writable,
filterOptions,
responseOptions,
requestStartTime,
);
return new Response(readable, { headers: responseHeaders });
} catch (error) {
return new Response(`Error processing JSON: ${error.message}`, {
status: 500,
});
}
},
};
function parseRequest(request: Request): RequestParams {
const url = new URL(request.url);
// Extract the JSON URL from the path
const jsonUrl = decodeURIComponent(url.pathname.slice(1));
// Parse filter options
const filterOptions: FilterOptions = {
omitFirstSegment: url.searchParams.get("omitFirstSegment") === "true",
omitBinary: url.searchParams.get("omitBinary") === "true",
enableFuzzyMatching: url.searchParams.get("enableFuzzyMatching") === "true",
rawUrlPrefix: url.searchParams.get("rawUrlPrefix"),
basePath: url.searchParams.getAll("basePath"),
pathPatterns: url.searchParams.getAll("pathPatterns"),
excludePathPatterns: url.searchParams.getAll("excludePathPatterns"),
maxFileSize: parseMaxFileSize(url.searchParams.get("maxFileSize")),
};
// Prepare response options
const responseOptions: ResponseOptions = {
boundary: `----WebKitFormBoundary${generateRandomString(16)}`,
isBrowser: isBrowserRequest(request),
authHeader: request.headers.get("x-source-authorization"),
};
return { jsonUrl, filterOptions, responseOptions };
}
function createErrorResponse(response: Response, jsonUrl: string): Response {
return new Response(
`----\nIngestjson: Failed to fetch JSON: URL=${jsonUrl}\n\n${
response.status
} ${response.statusText}\n\n${response.text()}\n\n-----`,
{ status: response.status },
);
}
// Helper functions for path normalization
const prependSlash = (path: string) =>
path.startsWith("/") ? path : "/" + path;
const surroundSlash = (path: string) =>
path.endsWith("/") ? prependSlash(path) : prependSlash(path) + "/";
const withoutSlash = (path: string) =>
path.startsWith("/") ? path.slice(1) : path;
/**
* Simple fuzzy matching function that works similarly to VS Code's fuzzy search
*/
function fuzzyMatch(pattern: string, str: string): boolean {
// Convert both strings to lowercase for case-insensitive matching
const lowerPattern = pattern.toLowerCase();
const lowerStr = str.toLowerCase();
let patternIdx = 0;
let strIdx = 0;
// Try to match all characters in the pattern in sequence
while (patternIdx < lowerPattern.length && strIdx < lowerStr.length) {
// If characters match, advance pattern index
if (lowerPattern[patternIdx] === lowerStr[strIdx]) {
patternIdx++;
}
// Always advance string index
strIdx++;
}
// If we've gone through the entire pattern, it's a match
return patternIdx === lowerPattern.length;
}
function parseMaxFileSize(maxFileSizeParam: string | null): number | undefined {
if (!maxFileSizeParam) {
return undefined;
}
const parsedSize = Number(maxFileSizeParam);
return !isNaN(parsedSize) ? parsedSize : undefined;
}
// Check if the request is authenticated
function isAuthenticated(request: Request, credentials: string): boolean {
const authHeader = request.headers.get("Authorization");
if (!authHeader) return false;
if (authHeader.startsWith("Basic ")) {
const base64Credentials = authHeader.slice(6);
return base64Credentials === btoa(credentials);
}
return false;
}
// Check if request is from a browser
function isBrowserRequest(request: Request): boolean {
const userAgent = request.headers.get("User-Agent") || "";
return /Mozilla|Chrome|Safari|Firefox|Edge/.test(userAgent);
}
// Generate a random string for the boundary
function generateRandomString(length: number): string {
const characters =
"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789";
let result = "";
const charactersLength = characters.length;
for (let i = 0; i < length; i++) {
result += characters.charAt(Math.floor(Math.random() * charactersLength));
}
return result;
}
// Process file path for omitFirstSegment option
function processFilePath(fileName: string, omitFirstSegment: boolean): string {
if (!omitFirstSegment) return fileName;
const parts = fileName.split("/");
if (parts.length <= 1) return fileName;
return "/" + parts.slice(1).join("/");
}
/**
* Precompile picomatch patterns for faster matching
*/
function compileMatchers(options: FilterOptions): CompiledMatchers {
// Common picomatch options
const picoOptions = {
dot: true, // Match dotfiles
windows: false, // Use forward slashes (POSIX style)
};
// For each category, create separate matchers for normal patterns and basename patterns
const inclusionMatchers = {
normal: [] as Array<(path: string) => boolean>,
basename: [] as Array<(basename: string) => boolean>,
};
const exclusionMatchers = {
normal: [] as Array<(path: string) => boolean>,
basename: [] as Array<(basename: string) => boolean>,
};
// Process inclusion patterns
if (options.pathPatterns && options.pathPatterns.length > 0) {
for (const pattern of options.pathPatterns) {
if (pattern.startsWith("*")) {
// Compile basename matchers
inclusionMatchers.basename.push(picomatch(pattern, picoOptions));
} else if (!pattern.includes("*") && !pattern.includes("?")) {
// VSCode-like behavior for non-glob patterns
inclusionMatchers.normal.push(picomatch(`${pattern}/**`, picoOptions));
} else {
// Standard pattern matching
inclusionMatchers.normal.push(picomatch(pattern, picoOptions));
}
}
}
// Process exclusion patterns
if (options.excludePathPatterns && options.excludePathPatterns.length > 0) {
for (const pattern of options.excludePathPatterns) {
if (pattern.startsWith("*")) {
// Compile basename matchers
exclusionMatchers.basename.push(picomatch(pattern, picoOptions));
} else if (!pattern.includes("*") && !pattern.includes("?")) {
// VSCode-like behavior for non-glob patterns
exclusionMatchers.normal.push(picomatch(`${pattern}/**`, picoOptions));
} else {
// Standard pattern matching
exclusionMatchers.normal.push(picomatch(pattern, picoOptions));
}
}
}
return {
inclusionMatchers,
exclusionMatchers,
hasInclusion:
inclusionMatchers.normal.length > 0 ||
inclusionMatchers.basename.length > 0,
hasExclusion:
exclusionMatchers.normal.length > 0 ||
exclusionMatchers.basename.length > 0,
};
}
// Updated CompiledMatchers interface
interface CompiledMatchers {
inclusionMatchers: {
normal: Array<(path: string) => boolean>;
basename: Array<(basename: string) => boolean>;
};
exclusionMatchers: {
normal: Array<(path: string) => boolean>;
basename: Array<(basename: string) => boolean>;
};
hasInclusion: boolean;
hasExclusion: boolean;
}
/**
* Process the JSON data and convert it to multipart/form-data stream
*/
async function processJsonToMultipart(
jsonData: any,
output: WritableStream,
filterOptions: FilterOptions,
responseOptions: ResponseOptions,
requestStartTime: number,
): Promise {
const { omitFirstSegment, rawUrlPrefix, omitBinary } = filterOptions;
const { boundary } = responseOptions;
const writer = output.getWriter();
const encoder = new TextEncoder();
try {
// Determine file structure based on JSON shape
const fileEntries = extractFileEntries(jsonData);
// Compile matchers once
const matchers = compileMatchers(filterOptions);
// Process each file entry
for (const { path, content, contentType, isUrl, size } of fileEntries) {
try {
// Apply filtering
const filter = shouldFilter(
filterOptions,
matchers,
path,
false, // isDirectory
size,
);
console.log({ path, filter });
if (filter?.filter) {
// If filtered out, decide whether to include in response with filter info
if (filter.noCallback) {
continue;
}
// Start multipart section
await writer.write(encoder.encode(`--${boundary}\r\n`));
await writer.write(
encoder.encode(
`Content-Disposition: form-data; name="${path}"; filename="${path}"\r\n`,
),
);
await writer.write(
encoder.encode(`Content-Type: application/json\r\n`),
);
await writer.write(
encoder.encode(
`x-filter: ingestjson;${filter.status || "404"};${
filter.message || ""
}\r\n\r\n`,
),
);
await writer.write(encoder.encode("\r\n"));
continue;
}
// Process path according to options
const processedPath = processFilePath(path, omitFirstSegment);
const ext = processedPath.split(".").pop() || "json";
// Check if it's a binary file
const isBinary = binaryExtensions.includes(ext);
if (omitBinary && isBinary && !isUrl) {
continue;
}
// Start multipart section
await writer.write(encoder.encode(`--${boundary}\r\n`));
await writer.write(
encoder.encode(
`Content-Disposition: form-data; name="${processedPath}"; filename="${processedPath}"\r\n`,
),
);
await writer.write(encoder.encode(`Content-Type: ${contentType}\r\n`));
if (size !== undefined) {
await writer.write(encoder.encode(`Content-Length: ${size}\r\n`));
}
// Handle binary content with raw URL prefix
if ((isBinary || isUrl) && rawUrlPrefix) {
const rawUrl = isUrl ? content : `${rawUrlPrefix}${processedPath}`;
await writer.write(encoder.encode(`x-url: ${rawUrl}\r\n`));
await writer.write(
encoder.encode(`Content-Transfer-Encoding: binary\r\n\r\n`),
);
await writer.write(encoder.encode("\r\n"));
continue;
}
// For binary content without rawUrlPrefix but with omitBinary
if (omitBinary && isBinary) {
await writer.write(
encoder.encode(`Content-Transfer-Encoding: binary\r\n\r\n`),
);
await writer.write(encoder.encode("\r\n"));
continue;
}
// For content that needs to be streamed
if (isUrl && !rawUrlPrefix) {
try {
// Fetch the content from the URL
const contentResponse = await fetch(content);
if (!contentResponse.ok) {
throw new Error(
`Failed to fetch content: ${contentResponse.status}`,
);
}
const contentData = await contentResponse.arrayBuffer();
const contentUint8 = new Uint8Array(contentData);
// Calculate hash
const hashBuffer = await crypto.subtle.digest(
"SHA-256",
contentUint8,
);
const hashArray = Array.from(new Uint8Array(hashBuffer));
const hashHex = hashArray
.map((b) => b.toString(16).padStart(2, "0"))
.join("");
await writer.write(encoder.encode(`x-file-hash: ${hashHex}\r\n`));
await writer.write(
encoder.encode(`Content-Transfer-Encoding: binary\r\n\r\n`),
);
// Write the content
await writer.write(contentUint8);
await writer.write(encoder.encode("\r\n"));
} catch (error) {
// If there's an error fetching content, include error info
await writer.write(
encoder.encode(`x-error: ${error.message}\r\n\r\n`),
);
await writer.write(encoder.encode("\r\n"));
}
continue;
}
// For direct content (not URL)
let contentBuffer: Uint8Array;
let hash: string | undefined;
if (typeof content === "string") {
contentBuffer = encoder.encode(content);
} else if (content instanceof Uint8Array) {
contentBuffer = content;
} else {
// For JSON data, stringify it
contentBuffer = encoder.encode(
typeof content === "object"
? JSON.stringify(content, null, 2)
: String(content),
);
}
// Calculate hash
if (contentBuffer) {
const hashBuffer = await crypto.subtle.digest(
"SHA-256",
contentBuffer,
);
const hashArray = Array.from(new Uint8Array(hashBuffer));
hash = hashArray.map((b) => b.toString(16).padStart(2, "0")).join("");
}
if (hash) {
await writer.write(encoder.encode(`x-file-hash: ${hash}\r\n`));
}
const contentIsBinary = !isUtf8(contentBuffer);
await writer.write(
encoder.encode(
`Content-Transfer-Encoding: ${
contentIsBinary ? "binary" : "8bit"
}\r\n\r\n`,
),
);
// Write the content
await writer.write(contentBuffer);
await writer.write(encoder.encode("\r\n"));
} catch (error) {
console.error(`Error processing file ${path}:`, error);
}
}
// End the multipart form data
await writer.write(encoder.encode(`--${boundary}--\r\n`));
const totalProcessingTime = Date.now() - requestStartTime;
console.log({ totalProcessingTime });
} catch (error) {
console.error("Error processing JSON:", error);
} finally {
await writer.close();
}
}
// Extract file entries from JSON data based on its shape
function extractFileEntries(jsonData: any): Array<{
path: string;
content: any;
contentType: string;
isUrl: boolean;
size?: number;
}> {
const entries: Array<{
path: string;
content: any;
contentType: string;
isUrl: boolean;
size?: number;
}> = [];
// Check if the JSON has the specified shape
if (
jsonData &&
typeof jsonData === "object" &&
jsonData.files &&
typeof jsonData.files === "object"
) {
for (const [path, fileInfo] of Object.entries(jsonData.files)) {
// Process files according to the files notation
if (typeof fileInfo !== "object") continue;
const { type, content, url, contentType, size } = fileInfo as any;
const ext = path.split("/").pop()?.split(".")?.pop();
if (type === "binary" && url) {
entries.push({
path,
content: url,
contentType: contentType || map[ext || "bin"],
isUrl: true,
size,
});
} else if (type === "content" && content !== undefined) {
entries.push({
path,
content,
contentType: contentType || map[ext || "md"],
isUrl: false,
size: typeof content === "string" ? content.length : undefined,
});
}
}
} else if (jsonData && typeof jsonData === "object") {
// For other JSON objects, create files based on first-level entries
if (Array.isArray(jsonData)) {
// For arrays, use indices as filenames
jsonData.forEach((value, index) => {
const content =
typeof value === "object"
? JSON.stringify(value, null, 2)
: String(value);
const ext = typeof value === "object" ? "json" : "md";
let filename = String(index);
// Use id or slug if available
if (typeof value === "object" && value !== null) {
if (
"id" in value &&
(typeof value.id === "string" || typeof value.id === "number")
) {
filename = String((value as any).id);
} else if ("slug" in value && typeof value.slug === "string") {
filename = String((value as any).slug);
}
}
entries.push({
path: `/${filename}.${ext}`,
content,
contentType: "application/json",
isUrl: false,
size: content.length,
});
});
} else {
// For objects, use keys as filenames
for (const [key, value] of Object.entries(jsonData)) {
const content =
typeof value === "object"
? JSON.stringify(value, null, 2)
: String(value);
const filename = String(key);
const ext =
typeof value === "object" ? "json" : filename.split(".").pop()!;
const mime = typeof value === "object" ? "application/json" : map[ext];
const hasExt = !!map[filename.split(".").pop()!];
entries.push({
path: hasExt ? `/${filename}` : `/${filename}.${ext}`,
content,
contentType: mime || "text/markdown",
isUrl: false,
size: content.length,
});
}
}
}
return entries;
}
// Check if content is valid UTF-8
function isUtf8(data: Uint8Array | undefined): boolean {
if (!data) {
return false;
}
try {
const decoder = new TextDecoder("utf-8", { fatal: true, ignoreBOM: false });
decoder.decode(data);
return true;
} catch {
return false;
}
}
const shouldFilter = (
filterOptions: FilterOptions,
matchers: CompiledMatchers,
fileName: string,
isDirectory: boolean,
size?: number,
): {
filter: boolean;
noCallback?: boolean;
status?: string;
message?: string;
} => {
if (isDirectory) return { filter: true, noCallback: true }; // Skip directories
const {
omitFirstSegment,
basePath,
maxFileSize,
omitBinary,
rawUrlPrefix,
enableFuzzyMatching,
pathPatterns,
excludePathPatterns,
} = filterOptions;
// Process the path with omitFirstSegment if needed
const processedPath = omitFirstSegment
? processFilePath(fileName, true)
: fileName;
// Check maxFileSize filter
if (maxFileSize !== undefined && size !== undefined && size > maxFileSize) {
return { filter: true, status: "413", message: "Content too large" };
}
const ext = processedPath.split(".").pop()!;
if (omitBinary && !rawUrlPrefix && binaryExtensions.includes(ext)) {
return {
filter: true,
status: "415",
message: "File has binary extension",
};
}
// basepath filter should not be applied, basepath is used for source
// Check base path filter
// if (basePath && basePath.length > 0) {
// const matchesBasePath = basePath.some((base) => {
// // Normalize base path and filename for directory matching
// const normalizedBase = surroundSlash(base);
// const normalizedFilename = surroundSlash(processedPath);
// return normalizedFilename.startsWith(normalizedBase);
// });
// if (!matchesBasePath) {
// return { filter: true, status: "404", message: "No basePath matched" };
// }
// }
// Extract basename once for potential basename pattern matching
const basename = processedPath.split("/").pop() || "";
const normalizedPath = withoutSlash(processedPath);
// Apply inclusion patterns if defined
let included = true;
if (
matchers.hasInclusion ||
(enableFuzzyMatching && pathPatterns && pathPatterns.length > 0)
) {
// Check normal patterns from picomatch
const matchesNormalPattern = matchers.inclusionMatchers.normal.some(
(matcher) => matcher(normalizedPath),
);
// Check basename patterns from picomatch
const matchesBasenamePattern = matchers.inclusionMatchers.basename.some(
(matcher) => matcher(basename),
);
// Apply fuzzy matching directly to path patterns if enabled
const matchesFuzzyPattern =
enableFuzzyMatching && pathPatterns
? pathPatterns.some((pattern) => {
// Only apply fuzzy matching to non-glob patterns
if (!pattern.includes("*") && !pattern.includes("?")) {
return fuzzyMatch(pattern, normalizedPath);
}
return false;
})
: false;
// File is included if it matches any pattern
included =
matchesNormalPattern || matchesBasenamePattern || matchesFuzzyPattern;
}
// If not included, no need to check exclusion
if (!included) {
return {
filter: true,
status: "404",
message: "Not included in path patterns",
};
}
// Apply exclusion patterns
if (
matchers.hasExclusion ||
(enableFuzzyMatching &&
excludePathPatterns &&
excludePathPatterns.length > 0)
) {
// Check normal patterns from picomatch
const matchesNormalExcludePattern = matchers.exclusionMatchers.normal.some(
(matcher) => matcher(normalizedPath),
);
// Check basename patterns from picomatch
const matchesBasenameExcludePattern =
matchers.exclusionMatchers.basename.some((matcher) => matcher(basename));
// Apply fuzzy matching directly to exclude path patterns if enabled
const matchesFuzzyExcludePattern =
enableFuzzyMatching && excludePathPatterns
? excludePathPatterns.some((pattern) => {
// Only apply fuzzy matching to non-glob patterns
if (!pattern.includes("*") && !pattern.includes("?")) {
return fuzzyMatch(pattern, normalizedPath);
}
return false;
})
: false;
// File is excluded if it matches any exclusion pattern
const excluded =
matchesNormalExcludePattern ||
matchesBasenameExcludePattern ||
matchesFuzzyExcludePattern;
// If excluded, it takes precedence over inclusion
if (excluded) {
return {
filter: true,
status: "404",
message: "Excluded by excludePathPatterns",
};
}
}
// If we reach this point, the file should be processed
return { filter: false };
};
```
## /uithub.ingestjson/package.json
```json path="/uithub.ingestjson/package.json"
{
"dependencies": {
"binary-extensions": "^3.0.0",
"fflate": "^0.8.2",
"picomatch": "^4.0.2"
},
"devDependencies": {
"@types/mime-db": "^1.43.5",
"@types/picomatch": "^4.0.0",
"mime-db": "^1.54.0"
}
}
```
## /uithub.ingestjson/test.js
```js path="/uithub.ingestjson/test.js"
/**
* Test script for the ingestjson module
*
* This test fetches a JSON document from a URL, processes it through
* the ingestjson module, and reports on the results.
*/
async function testJsonProcessor() {
console.time("Total execution time");
// JSON URL for testing
const jsonUrl = "https://cache.forgithub.com/oven-sh/bun/issues";
const CREDENTIALS = process.env.CREDENTIALS;
// Local or deployed server URL
const serverUrl = "http://localhost:3000"; // Change to "https://ingestjson.uithub.com" for production
// Construct the full URL with parameters
const url = new URL(serverUrl);
url.pathname = encodeURIComponent(jsonUrl);
// Optional: Add filter parameters
// url.searchParams.append("pathPatterns", "*.js");
// url.searchParams.append("pathPatterns", "*.json");
// url.searchParams.append("excludePathPatterns", "node_modules/**");
// url.searchParams.append("basePath", "some/path");
// url.searchParams.append("maxFileSize", "102400"); // 100 KB
console.log(`Fetching JSON from ${jsonUrl} and processing...`);
console.time("Fetch and process time");
try {
// Make the request
const response = await fetch(url.toString(), {
headers: {
Authorization: `Basic ${btoa(CREDENTIALS)}`,
// Optional: Add source authorization if the JSON requires auth
// "x-source-authorization": "Bearer your-token-here"
},
});
if (!response.ok) {
throw new Error(`Server responded with status: ${response.status}`);
}
// Process the response
const contentType = response.headers.get("content-type");
console.log(`Content-Type: ${contentType}`);
// Parse boundary from content type
const boundary = contentType.includes("boundary=")
? contentType.split("boundary=")[1].trim()
: "----WebKitFormBoundary";
console.log(`Using boundary: ${boundary}`);
// Get the full response as text
const responseText = await response.text();
// Count parts to determine number of files
const boundaryCount =
(responseText.match(new RegExp(`--${boundary}`, "g")) || []).length - 1; // Subtract 1 for the final boundary
console.timeEnd("Fetch and process time");
console.log(`Processed ${boundaryCount} files from the JSON document`);
// Print total size of the processed data
console.log(
`Total response size: ${(responseText.length / (1024 * 1024)).toFixed(
2,
)} MB`,
);
// Optional: Log file paths (first 10)
const fileNameRegex = /name="([^"]+)"/g;
const filePaths = [...responseText.matchAll(fileNameRegex)].map(
(match) => match[1],
);
console.log("\nSample of extracted files:");
filePaths.slice(0, 10).forEach((path) => console.log(` - ${path}`));
if (filePaths.length > 10) {
console.log(`... and ${filePaths.length - 10} more files`);
}
// Optional: Analyze file types
const fileExtensions = filePaths.map((path) => {
const parts = path.split(".");
return parts.length > 1 ? parts[parts.length - 1] : "unknown";
});
const fileTypeCount = fileExtensions.reduce((acc, ext) => {
acc[ext] = (acc[ext] || 0) + 1;
return acc;
}, {});
console.log("\nFile type distribution:");
Object.entries(fileTypeCount)
.sort((a, b) => b[1] - a[1])
.forEach(([ext, count]) => console.log(` - ${ext}: ${count} files`));
} catch (error) {
console.error("Error during test:", error);
}
console.timeEnd("Total execution time");
}
// Run the test
testJsonProcessor();
```
## /uithub.ingestjson/types.ts
```ts path="/uithub.ingestjson/types.ts"
export type Env = { CREDENTIALS: string };
export interface FilterOptions {
omitFirstSegment: boolean;
omitBinary: boolean;
enableFuzzyMatching: boolean;
rawUrlPrefix: string | null;
basePath: string[];
pathPatterns: string[];
excludePathPatterns: string[];
maxFileSize: number | undefined;
}
export interface ResponseOptions {
boundary: string;
isBrowser: boolean;
authHeader: string | null;
}
export interface RequestParams {
jsonUrl: string;
filterOptions: FilterOptions;
responseOptions: ResponseOptions;
}
```
## /uithub.ingestjson/wrangler.toml
```toml path="/uithub.ingestjson/wrangler.toml"
name = "uithub_ingestjson"
compatibility_date = "2025-04-01"
dev.port = 3000
route.pattern = "ingestjson.uithub.com"
route.custom_domain = true
main = "./main.ts"
# my feature!
limits.cpu_ms = 300000
```
## /uithub.ingestsql/.dev.vars.example
```example path="/uithub.ingestsql/.dev.vars.example"
CREDENTIALS=
```
## /uithub.ingestsql/.gitignore
```gitignore path="/uithub.ingestsql/.gitignore"
node_modules
.wrangler
package-lock.json
.dev.vars
.env
```
## /uithub.ingestsql/CHANGELOG.md
# First POC (2025-05-05)
I made this first version after being hyped up by https://x.com/benallfree who's working on a durable object file system (see https://x.com/benallfree/status/1919757983279677471). The first version had some problems though.
The problem with first version was that we couldn't leverage range and index match queries, which is very valuable but we are loosing. What if we could pass `/table-name[/indexed-column-name/match[/range]]`?
e.g. for https://github.com/janwilmake/hackernews-crawler/blob/main/main.ts, we get `/items/by/janwilmake`, `/items/time/202401010333/2025-010122233`, `/items/type/poll`, `/items/id/123456`, and we can even index more columns. This basically becomes a different source if done in this way, since we select just a single table, with or without match or range request, making it a lot faster as we can always use a range request easily by design of the URL.
Another big problem is that we can't know the table row size (can be up to 2mb for my source) and we don't currently stream responses, rather, we are sending it as JSON. A better solution potentially would be to be able to leverage the streaming rpc stuff, but extend that as API, such that we can determine early stop criteria on the other end, while streaming in every result asap. This could be done through SSE/JSONL events. This should probably be looked at a potential standard for.
# Second iteration (2025-05-07)
TODO:
- ✅ Created a new DORM version putting `exec` out of the DO boundary without altering the spec.
- ✅ Test with https://crawler.gcombinator.com to do a direct match on indexed column 'by'. How long does it take? **Query Duration: 0ms** but this doesn't work. But seems sub-second at least.
- ✅ Use GET https://dorm.wilmake.com/api/db/query/raw/QUERY as source.
This version relies on `dormroom@next` (v1) which implements `doStorage.sql.exec` outside of the durable object boundary, solving the second problem. The other big difference is that it can apply just a single query now, which is specified in the URL, along with filter params and a way to specify how the items turn into files.
This version allows full streaming without any query result size limitation, as well as full leveragability of SQL query efficiencies.
All in all, I think this is a better intermediate layer, where ultimately, the router that uses this as a source should apply the final path-to-source mapping. This final layer will have full control over how data is queried and how the files are then generated from it (to an extent). If more control is desired (e.g. we need multiple sources or we need to map results further), a FormData processing plugin can be made for that, ontop of ingestsql.
# TODO
- ✅ Specify how a domain router should work
- ✅ Try it with hackernews by tying its router to this instead, so I can has something similar to the actual hackernews routing.
- ✅ If that works, already make that accessible via `uuithub.com/news.ycombinator.com/*`
- ✅ Solve the plugin problem where that doesn't fit via a logical redirect (first segment becoming a plugin) if the path follows original HN. No redirect needed per se, as long as it has `pluginId` and `secondaryPageThing` while primary is empty. This way, navigation should work as desired.
- 🔥 HUGE HUGE HUGE. Once this works, I got the frontend for the DB, basically.
## /uithub.ingestsql/README.md
# ingestsql
Relevant context:
- https://raw.githubusercontent.com/janwilmake/uit/refs/heads/main/README.md
- https://uuithub.com/janwilmake/uit/tree/main/uithub.ingestzip?excludePathPatterns=*.html
- https://raw.githubusercontent.com/janwilmake/dorm/refs/heads/main/public/openapi.json
Input:
1. pathname contains a URL - `basePath`, which is a DORM Query URL. This sql server endpoint should follow the DORM API spec `GET /query/raw/QUERY` and support response content-type `application/x-ndjson` (StreamResponse)
2. a `x-source-authorization` (optional) which is passed as Authorization header to the sql server, if given
3. optional: a query param (can be string[]) `?itemTemplate` that contains an encoded string with the path for each item, where {property} will be replaced by the returned item property for all properties present.
4. optional: a query param (can be string[]) `?columnTemplate` that contains format `{columnName}:{pathTemplate}` that would put the result of the column in a file on the filled path. If the result is a URL, it will be set to x-url instead of the file contents, which will be made empty.
5. same query parameters as `ingestzip` (but not genignore)
Process:
- Executes the provided basePath with accept `application/x-ndjson` with the proper authorization header. Validates whether the response is valid or not (return early if not)
- Streams in the desired files based on `itemTemplate` and `columnTemplate`. Also applies query parameters in exactly the same way as in ingestzip
- Does not implement genignore.ts or genignore filter, this isn't needed for ingestsql
- output a FormData stream of files
It implements this in `types.ts` and `main.ts` and a spec of it `openapi.json` hosted at ingestsql.uithub.com
## /uithub.ingestsql/main.ts
```ts path="/uithub.ingestsql/main.ts"
import picomatch from "picomatch";
import binaryExtensions from "binary-extensions";
import map from "./public/ext-to-mime.json";
import {
Env,
FilterOptions,
RequestParams,
ResponseOptions,
StreamRecord,
ColumnTemplate,
ProcessedRow,
} from "./types";
export default {
async fetch(request: Request, env: Env, ctx: any): Promise {
// Parse the request parameters
const requestStartTime = Date.now();
const params = parseRequest(request);
const { sqlUrl, filterOptions, responseOptions } = params;
// Include timing info in response headers
const responseHeaders = new Headers({
"Content-Type": responseOptions.isBrowser
? `text/plain; boundary=${responseOptions.boundary}; charset=utf-8`
: `multipart/form-data; boundary=${responseOptions.boundary}`,
"Transfer-Encoding": "chunked",
});
// Validate the SQL URL
if (!sqlUrl) {
return new Response("No SQL URL provided", { status: 400 });
}
// Check authentication
if (!isAuthenticated(request, env.CREDENTIALS)) {
return new Response("Authentication required", {
status: 401,
headers: {
"WWW-Authenticate": 'Basic realm="SQL Access"',
},
});
}
try {
// Prepare headers for fetching SQL data
const headers = new Headers({
"User-Agent": "Cloudflare-Worker",
Accept: "application/x-ndjson",
});
if (responseOptions.authHeader) {
headers.set("Authorization", responseOptions.authHeader);
}
// Fetch the SQL data
const sqlResponse = await fetch(sqlUrl, { headers });
const initialResponseTime = Date.now() - requestStartTime;
responseHeaders.set(
"X-Initial-Response-Time-Ms",
initialResponseTime.toString(),
);
if (!sqlResponse.ok || !sqlResponse.body) {
return createErrorResponse(sqlResponse, params.sqlUrl);
}
// Process and stream the SQL contents
const { readable, writable } = new TransformStream();
// Start processing the SQL data in the background
processSqlToMultipart(
sqlResponse.body,
writable,
filterOptions,
params.responseOptions,
requestStartTime,
);
return new Response(readable, { headers: responseHeaders });
} catch (error) {
return new Response(`Error processing SQL: ${error.message}`, {
status: 500,
});
}
},
};
function parseRequest(request: Request): RequestParams {
const url = new URL(request.url);
// Extract the SQL URL from the path
const sqlUrl = decodeURIComponent(url.pathname.slice(1));
// Parse filter options
const filterOptions: FilterOptions = {
omitFirstSegment: url.searchParams.get("omitFirstSegment") === "true",
omitBinary: url.searchParams.get("omitBinary") === "true",
enableFuzzyMatching: url.searchParams.get("enableFuzzyMatching") === "true",
rawUrlPrefix: url.searchParams.get("rawUrlPrefix"),
basePath: url.searchParams.getAll("basePath"),
pathPatterns: url.searchParams.getAll("pathPatterns"),
excludePathPatterns: url.searchParams.getAll("excludePathPatterns"),
maxFileSize: parseMaxFileSize(url.searchParams.get("maxFileSize")),
itemTemplate: url.searchParams.getAll("itemTemplate"),
columnTemplate: url.searchParams.getAll("columnTemplate"),
};
// Prepare response options
const responseOptions: ResponseOptions = {
boundary: `----WebKitFormBoundary${generateRandomString(16)}`,
isBrowser: isBrowserRequest(request),
authHeader: request.headers.get("x-source-authorization"),
};
return { sqlUrl, filterOptions, responseOptions };
}
function createErrorResponse(response: Response, sqlUrl: string): Response {
return new Response(
`----\nIngestSQL: Failed to fetch SQL data: URL=${sqlUrl}\n\n${
response.status
} ${response.statusText}\n\n${response.text()}\n\n-----`,
{ status: response.status },
);
}
function parseMaxFileSize(maxFileSizeParam: string | null): number | undefined {
if (!maxFileSizeParam) {
return undefined;
}
const parsedSize = Number(maxFileSizeParam);
return !isNaN(parsedSize) ? parsedSize : undefined;
}
// Check if the request is authenticated
function isAuthenticated(request: Request, credentials: string): boolean {
const authHeader = request.headers.get("Authorization");
if (!authHeader) return false;
console.log({ credentials, authHeader });
if (authHeader.startsWith("Basic ")) {
const base64Credentials = authHeader.slice(6);
return base64Credentials === btoa(credentials);
}
return false;
}
// Check if request is from a browser
function isBrowserRequest(request: Request): boolean {
const userAgent = request.headers.get("User-Agent") || "";
return /Mozilla|Chrome|Safari|Firefox|Edge/.test(userAgent);
}
// Generate a random string for the boundary
function generateRandomString(length: number): string {
const characters =
"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789";
let result = "";
const charactersLength = characters.length;
for (let i = 0; i < length; i++) {
result += characters.charAt(Math.floor(Math.random() * charactersLength));
}
return result;
}
/**
* Parse column templates from the format "columnName:pathTemplate"
*/
function parseColumnTemplates(templates: string[]): ColumnTemplate[] {
const result: ColumnTemplate[] = [];
for (const template of templates) {
const colonIndex = template.indexOf(":");
if (colonIndex > 0) {
const columnName = template.substring(0, colonIndex);
const pathTemplate = template.substring(colonIndex + 1);
result.push({ columnName, pathTemplate });
}
}
return result;
}
/**
* Apply template to a row of data to create a filename
* Replaces {property} with the corresponding property value
*/
function applyTemplate(template: string, row: ProcessedRow): string {
let result = template;
// Replace column references
for (let i = 0; i < row.columns.length; i++) {
const columnName = row.columns[i];
const value = row.data[i];
if (value !== null && value !== undefined) {
// Replace both {columnName} and {index} patterns
result = result.replace(
new RegExp(`\\{${columnName}\\}`, "g"),
String(value),
);
}
}
// Replace row index
result = result.replace(/\{index\}/g, String(row.index));
return result;
}
/**
* Check if a path should be filtered based on the filter options
*/
function shouldFilter(
filterOptions: FilterOptions,
matchers: CompiledMatchers,
filePath: string,
fileSize?: number,
): {
filter: boolean;
status?: string;
message?: string;
} {
const {
omitFirstSegment,
basePath,
maxFileSize,
pathPatterns,
excludePathPatterns,
enableFuzzyMatching,
} = filterOptions;
// Process the path with omitFirstSegment if needed
const processedPath = omitFirstSegment
? processFilePath(filePath, true)
: filePath;
// Check maxFileSize filter
if (
maxFileSize !== undefined &&
fileSize !== undefined &&
fileSize > maxFileSize
) {
return { filter: true, status: "413", message: "Content too large" };
}
// Check base path filter
if (basePath && basePath.length > 0) {
const matchesBasePath = basePath.some((base) => {
// Normalize base path and filename for directory matching
const normalizedBase = surroundSlash(base);
const normalizedFilename = surroundSlash(processedPath);
return normalizedFilename.startsWith(normalizedBase);
});
if (!matchesBasePath) {
return { filter: true, status: "404", message: "No basePath matched" };
}
}
// Extract basename once for potential basename pattern matching
const basename = processedPath.split("/").pop() || "";
const normalizedPath = withoutSlash(processedPath);
// Apply inclusion patterns if defined
let included = true;
if (
matchers.hasInclusion ||
(enableFuzzyMatching && pathPatterns && pathPatterns.length > 0)
) {
// Check normal patterns from picomatch
const matchesNormalPattern = matchers.inclusionMatchers.normal.some(
(matcher) => matcher(normalizedPath),
);
// Check basename patterns from picomatch
const matchesBasenamePattern = matchers.inclusionMatchers.basename.some(
(matcher) => matcher(basename),
);
// Apply fuzzy matching directly to path patterns if enabled
const matchesFuzzyPattern =
enableFuzzyMatching && pathPatterns
? pathPatterns.some((pattern) => {
// Only apply fuzzy matching to non-glob patterns
if (!pattern.includes("*") && !pattern.includes("?")) {
return fuzzyMatch(pattern, normalizedPath);
}
return false;
})
: false;
// File is included if it matches any pattern
included =
matchesNormalPattern || matchesBasenamePattern || matchesFuzzyPattern;
}
// If not included, no need to check exclusion
if (!included) {
return {
filter: true,
status: "404",
message: "Not included in path patterns",
};
}
// Apply exclusion patterns
if (
matchers.hasExclusion ||
(enableFuzzyMatching &&
excludePathPatterns &&
excludePathPatterns.length > 0)
) {
// Check normal patterns from picomatch
const matchesNormalExcludePattern = matchers.exclusionMatchers.normal.some(
(matcher) => matcher(normalizedPath),
);
// Check basename patterns from picomatch
const matchesBasenameExcludePattern =
matchers.exclusionMatchers.basename.some((matcher) => matcher(basename));
// Apply fuzzy matching directly to exclude path patterns if enabled
const matchesFuzzyExcludePattern =
enableFuzzyMatching && excludePathPatterns
? excludePathPatterns.some((pattern) => {
// Only apply fuzzy matching to non-glob patterns
if (!pattern.includes("*") && !pattern.includes("?")) {
return fuzzyMatch(pattern, normalizedPath);
}
return false;
})
: false;
// File is excluded if it matches any exclusion pattern
const excluded =
matchesNormalExcludePattern ||
matchesBasenameExcludePattern ||
matchesFuzzyExcludePattern;
// If excluded, it takes precedence over inclusion
if (excluded) {
return {
filter: true,
status: "404",
message: "Excluded by excludePathPatterns",
};
}
}
// If we reach this point, the file should be processed
return { filter: false };
}
/**
* Process file path for omitFirstSegment option
*/
function processFilePath(fileName: string, omitFirstSegment: boolean): string {
if (!omitFirstSegment) return fileName;
const parts = fileName.split("/");
if (parts.length <= 1) return fileName;
return "/" + parts.slice(1).join("/");
}
/**
* Helper functions for path normalization
*/
const prependSlash = (path: string) =>
path.startsWith("/") ? path : "/" + path;
const surroundSlash = (path: string) =>
path.endsWith("/") ? prependSlash(path) : prependSlash(path) + "/";
const withoutSlash = (path: string) =>
path.startsWith("/") ? path.slice(1) : path;
/**
* Simple fuzzy matching function that works similarly to VS Code's fuzzy search
*/
function fuzzyMatch(pattern: string, str: string): boolean {
// Convert both strings to lowercase for case-insensitive matching
const lowerPattern = pattern.toLowerCase();
const lowerStr = str.toLowerCase();
let patternIdx = 0;
let strIdx = 0;
// Try to match all characters in the pattern in sequence
while (patternIdx < lowerPattern.length && strIdx < lowerStr.length) {
// If characters match, advance pattern index
if (lowerPattern[patternIdx] === lowerStr[strIdx]) {
patternIdx++;
}
// Always advance string index
strIdx++;
}
// If we've gone through the entire pattern, it's a match
return patternIdx === lowerPattern.length;
}
// Updated CompiledMatchers interface
interface CompiledMatchers {
inclusionMatchers: {
normal: Array<(path: string) => boolean>;
basename: Array<(basename: string) => boolean>;
};
exclusionMatchers: {
normal: Array<(path: string) => boolean>;
basename: Array<(basename: string) => boolean>;
};
hasInclusion: boolean;
hasExclusion: boolean;
}
/**
* Precompile picomatch patterns for faster matching
*/
function compileMatchers(options: FilterOptions): CompiledMatchers {
// Common picomatch options
const picoOptions = {
dot: true, // Match dotfiles
windows: false, // Use forward slashes (POSIX style)
};
// For each category, create separate matchers for normal patterns and basename patterns
const inclusionMatchers = {
normal: [] as Array<(path: string) => boolean>,
basename: [] as Array<(basename: string) => boolean>,
};
const exclusionMatchers = {
normal: [] as Array<(path: string) => boolean>,
basename: [] as Array<(basename: string) => boolean>,
};
// Process inclusion patterns
if (options.pathPatterns && options.pathPatterns.length > 0) {
for (const pattern of options.pathPatterns) {
if (pattern.startsWith("*")) {
// Compile basename matchers
inclusionMatchers.basename.push(picomatch(pattern, picoOptions));
} else if (!pattern.includes("*") && !pattern.includes("?")) {
// VSCode-like behavior for non-glob patterns
inclusionMatchers.normal.push(picomatch(`${pattern}/**`, picoOptions));
} else {
// Standard pattern matching
inclusionMatchers.normal.push(picomatch(pattern, picoOptions));
}
}
}
// Process exclusion patterns
if (options.excludePathPatterns && options.excludePathPatterns.length > 0) {
for (const pattern of options.excludePathPatterns) {
if (pattern.startsWith("*")) {
// Compile basename matchers
exclusionMatchers.basename.push(picomatch(pattern, picoOptions));
} else if (!pattern.includes("*") && !pattern.includes("?")) {
// VSCode-like behavior for non-glob patterns
exclusionMatchers.normal.push(picomatch(`${pattern}/**`, picoOptions));
} else {
// Standard pattern matching
exclusionMatchers.normal.push(picomatch(pattern, picoOptions));
}
}
}
return {
inclusionMatchers,
exclusionMatchers,
hasInclusion:
inclusionMatchers.normal.length > 0 ||
inclusionMatchers.basename.length > 0,
hasExclusion:
exclusionMatchers.normal.length > 0 ||
exclusionMatchers.basename.length > 0,
};
}
/**
* Determine if content is likely to be a URL
*/
function isUrl(str: string): boolean {
try {
const url = new URL(str);
return url.protocol === "http:" || url.protocol === "https:";
} catch {
return false;
}
}
/**
* Get file extension from path
*/
function getExtension(path: string): string {
return path.split(".").pop()?.toLowerCase() || "";
}
/**
* Get content type from file extension
*/
function getContentType(ext: string): string {
return (map[ext] || "application/octet-stream") as string;
}
/**
* Check if content is valid UTF-8
*/
function isUtf8(data: Uint8Array | undefined): boolean {
if (!data) {
return false;
}
try {
const decoder = new TextDecoder("utf-8", { fatal: true, ignoreBOM: false });
decoder.decode(data);
return true;
} catch {
return false;
}
}
// TextEncoder for string to Uint8Array conversion
const encoder = new TextEncoder();
/**
* Generate SHA-256 hash for content
*/
async function generateHash(content: Uint8Array): Promise {
const hashBuffer = await crypto.subtle.digest("SHA-256", content);
const hashArray = Array.from(new Uint8Array(hashBuffer));
return hashArray.map((b) => b.toString(16).padStart(2, "0")).join("");
}
/**
* Process SQL data stream to multipart/form-data
*/
async function processSqlToMultipart(
sqlStream: ReadableStream,
output: WritableStream,
filterOptions: FilterOptions,
responseOptions: ResponseOptions,
requestStartTime: number,
): Promise {
const { boundary } = responseOptions;
const writer = output.getWriter();
const reader = sqlStream.getReader();
const matchers = compileMatchers(filterOptions);
// Parse item and column templates
const itemTemplates = filterOptions.itemTemplate || [];
const columnTemplates = parseColumnTemplates(
filterOptions.columnTemplate || [],
);
let columns: string[] = [];
let rowCount = 0;
const textDecoder = new TextDecoder();
let buffer = "";
try {
while (true) {
const { done, value } = await reader.read();
if (done) {
// Process any remaining data in the buffer
if (buffer.trim()) {
await processRecord(JSON.parse(buffer.trim()));
}
break;
}
const chunk = textDecoder.decode(value, { stream: true });
buffer += chunk;
// Process complete lines
const lines = buffer.split("\n");
buffer = lines.pop() || ""; // Keep the last (potentially incomplete) line in the buffer
for (const line of lines) {
if (line.trim()) {
try {
const record: StreamRecord = JSON.parse(line);
await processRecord(record);
} catch (e) {
console.error("Error parsing JSON line:", e);
}
}
}
}
// End the multipart form data
await writer.write(encoder.encode(`--${boundary}--\r\n`));
const totalProcessingTime = Date.now() - requestStartTime;
console.log({ totalProcessingTime, rowsProcessed: rowCount });
} catch (error) {
console.error("Error processing SQL data:", error);
} finally {
await writer.close();
}
async function processRecord(record: StreamRecord): Promise {
if (record.type === "columns" && Array.isArray(record.data)) {
columns = record.data as string[];
} else if (record.type === "row" && Array.isArray(record.data)) {
const rowData = record.data as any[];
const row: ProcessedRow = {
index: rowCount++,
data: rowData,
columns: columns,
};
// Process item templates
for (const template of itemTemplates) {
if (template) {
const filePath = applyTemplate(template, row);
await writeFile(filePath, row);
}
}
// Process column templates
for (const template of columnTemplates) {
const columnIndex = columns.indexOf(template.columnName);
if (columnIndex >= 0 && columnIndex < rowData.length) {
const value = rowData[columnIndex];
if (value !== null && value !== undefined) {
const filePath = applyTemplate(template.pathTemplate, row);
await writeColumnFile(filePath, value);
}
}
}
}
}
/**
* Write a file from row data
*/
async function writeFile(filePath: string, row: ProcessedRow): Promise {
// Check if this file should be filtered
const filterResult = shouldFilter(filterOptions, matchers, filePath);
if (filterResult.filter) {
// Write filtered file with empty content and filter header
await writeFilteredFile(filePath, filterResult);
return;
}
// Convert row to JSON
const content = JSON.stringify(
Object.fromEntries(row.columns.map((col, i) => [col, row.data[i]])),
null,
2,
);
const contentBytes = encoder.encode(content);
// Calculate hash
const hash = await generateHash(contentBytes);
// Get file extension and content type
const ext = getExtension(filePath);
const contentType = getContentType(ext);
// Start multipart section
await writer.write(encoder.encode(`--${boundary}\r\n`));
await writer.write(
encoder.encode(
`Content-Disposition: form-data; name="${filePath}"; filename="${filePath}"\r\n`,
),
);
await writer.write(encoder.encode(`Content-Type: ${contentType}\r\n`));
await writer.write(
encoder.encode(`Content-Length: ${contentBytes.length}\r\n`),
);
await writer.write(encoder.encode(`x-file-hash: ${hash}\r\n`));
await writer.write(
encoder.encode(`Content-Transfer-Encoding: 8bit\r\n\r\n`),
);
await writer.write(contentBytes);
await writer.write(encoder.encode("\r\n"));
}
/**
* Write a file from a single column value
*/
async function writeColumnFile(filePath: string, value: any): Promise {
// Check if this file should be filtered
const filterResult = shouldFilter(filterOptions, matchers, filePath);
if (filterResult.filter) {
// Write filtered file with empty content and filter header
await writeFilteredFile(filePath, filterResult);
return;
}
// Check if the value is a URL
const isUrlValue = typeof value === "string" && isUrl(value);
// Get file extension and content type
const ext = getExtension(filePath);
const contentType = getContentType(ext);
const isBinaryExt = binaryExtensions.includes(ext);
// Handle binary files with rawUrlPrefix or URL values
if (
(filterOptions.omitBinary && isBinaryExt) ||
(isUrlValue && filterOptions.rawUrlPrefix)
) {
await writeEmptyFile(
filePath,
contentType,
isUrlValue ? value : `${filterOptions.rawUrlPrefix}${filePath}`,
);
return;
}
// Convert value to string or JSON if object
let content: string;
if (typeof value === "object") {
content = JSON.stringify(value, null, 2);
} else {
content = String(value);
}
const contentBytes = encoder.encode(content);
// Calculate hash
const hash = await generateHash(contentBytes);
// Start multipart section
await writer.write(encoder.encode(`--${boundary}\r\n`));
await writer.write(
encoder.encode(
`Content-Disposition: form-data; name="${filePath}"; filename="${filePath}"\r\n`,
),
);
await writer.write(encoder.encode(`Content-Type: ${contentType}\r\n`));
await writer.write(
encoder.encode(`Content-Length: ${contentBytes.length}\r\n`),
);
await writer.write(encoder.encode(`x-file-hash: ${hash}\r\n`));
if (isUrlValue) {
await writer.write(encoder.encode(`x-url: ${value}\r\n`));
}
await writer.write(
encoder.encode(`Content-Transfer-Encoding: 8bit\r\n\r\n`),
);
await writer.write(contentBytes);
await writer.write(encoder.encode("\r\n"));
}
/**
* Write a filtered file with status and message
*/
async function writeFilteredFile(
filePath: string,
filterResult: { filter: boolean; status?: string; message?: string },
): Promise {
const ext = getExtension(filePath);
const contentType = getContentType(ext);
await writer.write(encoder.encode(`--${boundary}\r\n`));
await writer.write(
encoder.encode(
`Content-Disposition: form-data; name="${filePath}"; filename="${filePath}"\r\n`,
),
);
await writer.write(encoder.encode(`Content-Type: ${contentType}\r\n`));
await writer.write(
encoder.encode(
`x-filter: ingestsql;${filterResult.status || "404"};${
filterResult.message || ""
}\r\n`,
),
);
await writer.write(
encoder.encode(`Content-Transfer-Encoding: 8bit\r\n\r\n`),
);
await writer.write(encoder.encode("\r\n"));
}
/**
* Write an empty file with URL reference
*/
async function writeEmptyFile(
filePath: string,
contentType: string,
url: string,
): Promise {
await writer.write(encoder.encode(`--${boundary}\r\n`));
await writer.write(
encoder.encode(
`Content-Disposition: form-data; name="${filePath}"; filename="${filePath}"\r\n`,
),
);
await writer.write(encoder.encode(`Content-Type: ${contentType}\r\n`));
await writer.write(encoder.encode(`x-url: ${url}\r\n`));
await writer.write(
encoder.encode(`Content-Transfer-Encoding: binary\r\n\r\n`),
);
await writer.write(encoder.encode("\r\n"));
}
}
```
## /uithub.ingestsql/package.json
```json path="/uithub.ingestsql/package.json"
{
"dependencies": {
"binary-extensions": "^3.0.0",
"picomatch": "^4.0.2"
},
"devDependencies": {
"@types/mime-db": "^1.43.5",
"@types/picomatch": "^4.0.0",
"mime-db": "^1.54.0"
}
}
```
## /uithub.ingestsql/public/default-genignore.txt
# Package manager locks and metadata
package-lock.json
*.lock
pnpm-lock.yaml
npm-shrinkwrap.json
node_modules
.pnp.*
.yarn/*
.npm
# Build artifacts and output directories
dist/
build/
out/
.output
.nuxt
.next
.cache
.parcel-cache
.webpack
.serverless
.gradle
target/
bin/
# Dependency directories
bower_components/
jspm_packages/
web_modules/
# Environment and local configuration files
.env
.env.*
.envrc
.vercel
.venv
env/
venv/
ENV/
.local
# Editor directories and files
.idea/
.vscode/
.vs/
*.swp
*.swo
*~
.DS_Store
._*
Thumbs.db
# Debug logs
npm-debug.log*
yarn-debug.log*
yarn-error.log*
lerna-debug.log*
.pnpm-debug.log*
debug.log
# Test coverage
coverage/
.nyc_output
.coverage
htmlcov/
.tox/
.pytest_cache/
__pycache__/
*.py[cod]
*$py.class
.hypothesis/
# Temporary files
*.tmp
*.temp
tmp/
temp/
.temp/
.tmp/
# IDE and tooling configuration
.eslintcache
.stylelintcache
.prettierignore
.commitlintrc
.editorconfig
.circleci/
.github/workflows/
.gitlab-ci.yml
.travis.yml
.codeclimate.yml
.dependabot/
renovate.json
# Documentation artifacts
docs/_build/
site/
.docusaurus
storybook-static/
# Minified files (often not useful for context)
*.min.js
*.min.css
# Generated assets & maps
*.map
*.d.ts
*.tsbuildinfo
# Auto-generated files
LICENSE
.gitattributes
## /uithub.ingestsql/public/openapi.json
```json path="/uithub.ingestsql/public/openapi.json"
{
"openapi": "3.0.3",
"info": {
"title": "IngestSQL API",
"description": "An API service that fetches SQL query results from DORM (Durable Object Relational Mapper) and converts them to multipart/form-data streams with various filtering and templating options.",
"version": "1.0.0",
"contact": {
"name": "UIThub",
"url": "https://ingestsql.uithub.com"
}
},
"servers": [
{
"url": "https://ingestsql.uithub.com",
"description": "Production server"
}
],
"paths": {
"/{sqlUrl}": {
"get": {
"summary": "Fetch and process SQL query results",
"description": "Fetches SQL query results from a DORM API endpoint, processes the contents using templates, and returns a multipart/form-data stream with the filtered files.",
"parameters": [
{
"name": "sqlUrl",
"in": "path",
"description": "URL-encoded path to the SQL query endpoint (DORM API GET /query/raw/QUERY)",
"required": true,
"schema": {
"type": "string"
}
},
{
"name": "omitFirstSegment",
"in": "query",
"description": "Remove the first segment from file paths in the output",
"required": false,
"schema": {
"type": "boolean",
"default": false
}
},
{
"name": "omitBinary",
"in": "query",
"description": "Skip binary files in the output",
"required": false,
"schema": {
"type": "boolean",
"default": false
}
},
{
"name": "enableFuzzyMatching",
"in": "query",
"description": "Enable fuzzy matching for path patterns",
"required": false,
"schema": {
"type": "boolean",
"default": false
}
},
{
"name": "rawUrlPrefix",
"in": "query",
"description": "Prefix to use for raw URLs for binary files (if set, binary files are referenced by URL instead of including content)",
"required": false,
"schema": {
"type": "string"
}
},
{
"name": "basePath",
"in": "query",
"description": "Base path(s) to filter files by. Only files under these paths will be included",
"required": false,
"schema": {
"type": "array",
"items": {
"type": "string"
}
},
"style": "form",
"explode": true
},
{
"name": "pathPatterns",
"in": "query",
"description": "Glob pattern(s) to include files matching these patterns. Supports picomatch syntax",
"required": false,
"schema": {
"type": "array",
"items": {
"type": "string"
}
},
"style": "form",
"explode": true
},
{
"name": "excludePathPatterns",
"in": "query",
"description": "Glob pattern(s) to exclude files matching these patterns. Supports picomatch syntax",
"required": false,
"schema": {
"type": "array",
"items": {
"type": "string"
}
},
"style": "form",
"explode": true
},
{
"name": "maxFileSize",
"in": "query",
"description": "Maximum file size in bytes to include",
"required": false,
"schema": {
"type": "integer",
"format": "int64"
}
},
{
"name": "itemTemplate",
"in": "query",
"description": "Template string for generating file paths from row data. Use {property} syntax to reference columns by name and {index} for row number.",
"required": false,
"schema": {
"type": "array",
"items": {
"type": "string"
}
},
"style": "form",
"explode": true,
"example": ["/records/{id}/record.json", "/items/item_{index}.json"]
},
{
"name": "columnTemplate",
"in": "query",
"description": "Template string for generating file paths from individual column values. Format is 'columnName:pathTemplate'.",
"required": false,
"schema": {
"type": "array",
"items": {
"type": "string"
}
},
"style": "form",
"explode": true,
"example": ["name:/names/{id}.txt", "avatar:/avatars/{id}.png"]
}
],
"security": [
{
"basicAuth": []
}
],
"responses": {
"200": {
"description": "SQL data successfully processed and streamed",
"content": {
"multipart/form-data": {
"schema": {
"type": "object"
}
},
"text/plain": {
"schema": {
"type": "string"
}
}
}
},
"400": {
"description": "Bad request, missing SQL URL",
"content": {
"text/plain": {
"schema": {
"type": "string"
}
}
}
},
"401": {
"description": "Unauthorized, authentication required",
"headers": {
"WWW-Authenticate": {
"schema": {
"type": "string",
"example": "Basic realm=\"SQL Access\""
}
}
},
"content": {
"text/plain": {
"schema": {
"type": "string"
}
}
}
},
"500": {
"description": "Server error processing the SQL data",
"content": {
"text/plain": {
"schema": {
"type": "string"
}
}
}
}
}
}
}
},
"components": {
"securitySchemes": {
"basicAuth": {
"type": "http",
"scheme": "basic",
"description": "Basic authentication with credentials"
}
},
"headers": {
"x-source-authorization": {
"description": "Optional authorization header to forward to the SQL source",
"schema": {
"type": "string"
}
}
},
"schemas": {
"FormDataFile": {
"type": "object",
"properties": {
"content": {
"type": "string",
"format": "binary",
"description": "File content"
},
"x-file-hash": {
"type": "string",
"description": "SHA-256 hash of the file content"
},
"x-url": {
"type": "string",
"description": "URL to access the raw file (only present for binary files when rawUrlPrefix is provided or for URL column values)"
},
"x-filter": {
"type": "string",
"description": "Filter information if the file was filtered out but still included in the response"
}
}
},
"StreamRecord": {
"oneOf": [
{
"type": "object",
"properties": {
"type": {
"type": "string",
"enum": ["columns"],
"description": "Indicates this record contains column names"
},
"data": {
"type": "array",
"items": {
"type": "string"
},
"description": "Array of column names"
}
},
"required": ["type", "data"]
},
{
"type": "object",
"properties": {
"type": {
"type": "string",
"enum": ["row"],
"description": "Indicates this record contains a single row of data"
},
"data": {
"type": "array",
"items": {},
"description": "Array of values in the row"
}
},
"required": ["type", "data"]
},
{
"type": "object",
"properties": {
"type": {
"type": "string",
"enum": ["meta"],
"description": "Indicates this record contains metadata about the query"
},
"data": {
"type": "object",
"properties": {
"rows_read": {
"type": "integer",
"description": "Number of rows read by the query"
},
"rows_written": {
"type": "integer",
"description": "Number of rows written by the query"
}
},
"required": ["rows_read", "rows_written"]
}
},
"required": ["type", "data"]
},
{
"type": "object",
"properties": {
"type": {
"type": "string",
"enum": ["error"],
"description": "Indicates an error occurred during streaming"
},
"error": {
"type": "string",
"description": "Error message"
}
},
"required": ["type", "error"]
}
]
}
}
},
"tags": [
{
"name": "SQL Processing",
"description": "Operations related to fetching and processing SQL query results"
}
]
}
```
## /uithub.ingestsql/types.ts
```ts path="/uithub.ingestsql/types.ts"
export type Env = { CREDENTIALS: string };
export interface FilterOptions {
omitFirstSegment: boolean;
omitBinary: boolean;
enableFuzzyMatching: boolean;
rawUrlPrefix: string | null;
basePath: string[];
pathPatterns: string[];
excludePathPatterns: string[];
maxFileSize: number | undefined;
itemTemplate?: string[];
columnTemplate?: string[];
}
export interface ResponseOptions {
boundary: string;
isBrowser: boolean;
authHeader: string | null;
}
export interface RequestParams {
sqlUrl: string;
filterOptions: FilterOptions;
responseOptions: ResponseOptions;
}
export interface StreamRecord {
type: "columns" | "row" | "meta" | "error";
data?: any[] | { rows_read: number; rows_written: number };
error?: string;
}
export interface ColumnTemplate {
columnName: string;
pathTemplate: string;
}
export interface ProcessedRow {
index: number;
data: any[];
columns: string[];
}
```
## /uithub.ingestsql/wrangler.toml
```toml path="/uithub.ingestsql/wrangler.toml"
name = "uithub_ingestsql"
compatibility_date = "2025-04-01"
dev.port = 3000
route.pattern = "ingestsql.uithub.com"
route.custom_domain = true
main = "./main.ts"
assets.directory = "./public"
# my feature!
limits.cpu_ms = 300000
```
## /uithub.ingesttar/.gitignore
```gitignore path="/uithub.ingesttar/.gitignore"
node_modules
.wrangler
package-lock.json
.dev.vars
.env
```
## /uithub.ingesttar/README.md
similarly to ingestzip, this parses tar files from first principles using Web Standards
## /uithub.ingesttar/genignore.ts
```ts path="/uithub.ingesttar/genignore.ts"
//@ts-ignore
import defaultGenignore from "./public/default-genignore.txt";
import { FilterOptions, ResponseOptions } from "./types";
import { findGenIgnoreInTar } from "./tarReader";
/**
* Parse the .genignore file content into an array of exclude patterns
*
* @param content The content of the .genignore file
* @returns Array of exclude patterns
*/
export function parseGenIgnore(content: string): string[] {
// Split by lines, trim whitespace, and filter out comments and empty lines
return content
.split("\n")
.map((line) => line.trim())
.filter((line) => line && !line.startsWith("#"));
}
/**
* Integrates the first pass for .genignore into the current process flow.
* Returns modified filter options with updated excludePathPatterns.
*/
export async function processWithGenIgnore(
tarUrl: string,
initialFilterOptions: FilterOptions,
responseOptions: ResponseOptions,
): Promise<{
updatedFilterOptions: FilterOptions;
tarResponse: Response;
}> {
// Prepare headers for fetching the TAR
const headers = new Headers({ "User-Agent": "Cloudflare-Worker" });
if (responseOptions.authHeader) {
headers.set("Authorization", responseOptions.authHeader);
}
// Make a clone of the initial filter options
const updatedFilterOptions = { ...initialFilterOptions };
// If we don't need to check for .genignore, just fetch the TAR once
if (!initialFilterOptions.genignore) {
console.log("Genignore not desired");
const tarResponse = await fetch(tarUrl, { headers });
return { updatedFilterOptions, tarResponse };
}
// First pass: Fetch the TAR to look for .genignore
const firstPassResponse = await fetch(tarUrl, { headers });
if (!firstPassResponse.ok || !firstPassResponse.body) {
// If the first pass fails, just return the response as is
return { updatedFilterOptions, tarResponse: firstPassResponse };
}
// Clone the body stream to keep the original for later use
const [firstPassStream, tarResponseBody] = firstPassResponse.body.tee();
// Create a response with the cloned body for returning later
const tarResponse = new Response(tarResponseBody, {
status: firstPassResponse.status,
statusText: firstPassResponse.statusText,
headers: firstPassResponse.headers,
});
try {
// Decompress gzip stream
const decompressedStream = firstPassStream.pipeThrough(
new DecompressionStream("gzip"),
);
// Look for .genignore in the TAR
const genIgnorePatterns =
(await findGenIgnoreInTar(decompressedStream)) ||
parseGenIgnore(defaultGenignore);
console.log({ genIgnorePatterns });
// unique and concatenate with provided ones
updatedFilterOptions.excludePathPatterns = Array.from(
new Set(
genIgnorePatterns.concat(updatedFilterOptions.excludePathPatterns),
),
);
} catch (error) {
console.error("Error during .genignore processing:", error);
// Fall back to default genignore patterns on error
updatedFilterOptions.excludePathPatterns = parseGenIgnore(defaultGenignore);
}
return { updatedFilterOptions, tarResponse };
}
```
## /uithub.ingesttar/globals.d.ts
```ts path="/uithub.ingesttar/globals.d.ts"
declare module "*.md" {
const content: string;
export default content;
}
declare module "*.html" {
const content: string;
export default content;
}
```
## /uithub.ingesttar/main.ts
```ts path="/uithub.ingesttar/main.ts"
// Import picomatch for glob pattern matching
import picomatch from "picomatch";
import map from "./public/ext-to-mime.json";
import binaryExtensions from "binary-extensions";
import { Env, FilterOptions, RequestParams, ResponseOptions } from "./types";
import { processWithGenIgnore } from "./genignore";
import { processTarStream } from "./tarReader";
export default {
async fetch(request: Request, env: Env, ctx: any): Promise {
// Parse the request parameters
const requestStartTime = Date.now();
const params = parseRequest(request);
const {
tarUrl,
filterOptions: initialFilterOptions,
responseOptions,
} = params;
// Include timing info in response headers
const responseHeaders = new Headers({
"Content-Type": responseOptions.isBrowser
? `text/plain; boundary=${responseOptions.boundary}; charset=utf-8`
: `multipart/form-data; boundary=${responseOptions.boundary}`,
"Transfer-Encoding": "chunked",
});
// Validate the TAR URL
if (!tarUrl) {
return new Response("No TAR URL provided", { status: 400 });
}
// Check authentication
if (!isAuthenticated(request, env.CREDENTIALS)) {
return new Response("Authentication required", {
status: 401,
headers: {
"WWW-Authenticate": 'Basic realm="Archive Access"',
},
});
}
try {
// Process with GenIgnore - this does the first pass if needed
const { updatedFilterOptions, tarResponse } = await processWithGenIgnore(
tarUrl,
initialFilterOptions,
responseOptions,
);
console.log({ genignore: updatedFilterOptions.excludePathPatterns });
const initialResponseTime = Date.now() - requestStartTime;
console.log({ initialResponseTime });
responseHeaders.set(
"X-Initial-Response-Time-Ms",
initialResponseTime.toString(),
);
if (!tarResponse.ok || !tarResponse.body) {
return createErrorResponse(tarResponse, params.tarUrl);
}
// Process and stream the TAR contents
const { readable, writable } = new TransformStream();
console.log({ updatedFilterOptions });
// Start processing the TAR file in the background with updated filter options
processTarToMultipart(
tarResponse.body,
writable,
updatedFilterOptions, // Use the updated filter options with .genignore patterns
params.responseOptions,
requestStartTime,
);
return new Response(readable, { headers: responseHeaders });
} catch (error) {
return new Response(`Error processing TAR: ${error.message}`, {
status: 500,
});
}
},
};
function parseRequest(request: Request): RequestParams {
const url = new URL(request.url);
// Extract the TAR URL from the path
const tarUrl = decodeURIComponent(url.pathname.slice(1));
// Parse filter options
const filterOptions: FilterOptions = {
genignore: url.searchParams.get("genignore") !== "false",
omitFirstSegment: url.searchParams.get("omitFirstSegment") === "true",
omitBinary: url.searchParams.get("omitBinary") === "true",
enableFuzzyMatching: url.searchParams.get("enableFuzzyMatching") === "true",
rawUrlPrefix: url.searchParams.get("rawUrlPrefix"),
basePath: url.searchParams.getAll("basePath"),
pathPatterns: url.searchParams.getAll("pathPatterns"),
excludePathPatterns: url.searchParams.getAll("excludePathPatterns"),
maxFileSize: parseMaxFileSize(url.searchParams.get("maxFileSize")),
};
// Prepare response options
const responseOptions: ResponseOptions = {
boundary: `----WebKitFormBoundary${generateRandomString(16)}`,
isBrowser: isBrowserRequest(request),
authHeader: request.headers.get("x-source-authorization"),
};
return { tarUrl, filterOptions, responseOptions };
}
function createErrorResponse(response: Response, tarUrl: string): Response {
return new Response(
`----\nIngesttar: Failed to fetch TAR: URL=${tarUrl}\n\n${
response.status
} ${response.statusText}\n\n${response.text()}\n\n-----`,
{ status: response.status },
);
}
// Helper functions for path normalization
const prependSlash = (path: string) =>
path.startsWith("/") ? path : "/" + path;
const surroundSlash = (path: string) =>
path.endsWith("/") ? prependSlash(path) : prependSlash(path) + "/";
const withoutSlash = (path: string) =>
path.startsWith("/") ? path.slice(1) : path;
/**
* Simple fuzzy matching function that works similarly to VS Code's fuzzy search
*/
function fuzzyMatch(pattern: string, str: string): boolean {
// Convert both strings to lowercase for case-insensitive matching
const lowerPattern = pattern.toLowerCase();
const lowerStr = str.toLowerCase();
let patternIdx = 0;
let strIdx = 0;
// Try to match all characters in the pattern in sequence
while (patternIdx < lowerPattern.length && strIdx < lowerStr.length) {
// If characters match, advance pattern index
if (lowerPattern[patternIdx] === lowerStr[strIdx]) {
patternIdx++;
}
// Always advance string index
strIdx++;
}
// If we've gone through the entire pattern, it's a match
return patternIdx === lowerPattern.length;
}
function parseMaxFileSize(maxFileSizeParam: string | null): number | undefined {
if (!maxFileSizeParam) {
return undefined;
}
const parsedSize = Number(maxFileSizeParam);
return !isNaN(parsedSize) ? parsedSize : undefined;
}
// Check if the request is authenticated
function isAuthenticated(request: Request, credentials: string): boolean {
const authHeader = request.headers.get("Authorization");
if (!authHeader) return false;
if (authHeader.startsWith("Basic ")) {
const base64Credentials = authHeader.slice(6);
return base64Credentials === btoa(credentials);
}
return false;
}
// Check if request is from a browser
function isBrowserRequest(request: Request): boolean {
const userAgent = request.headers.get("User-Agent") || "";
return /Mozilla|Chrome|Safari|Firefox|Edge/.test(userAgent);
}
// Generate a random string for the boundary
function generateRandomString(length: number): string {
const characters =
"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789";
let result = "";
const charactersLength = characters.length;
for (let i = 0; i < length; i++) {
result += characters.charAt(Math.floor(Math.random() * charactersLength));
}
return result;
}
// Process file path for omitFirstSegment option
function processFilePath(fileName: string, omitFirstSegment: boolean): string {
if (!omitFirstSegment) return fileName;
const parts = fileName.split("/");
if (parts.length <= 1) return fileName;
return "/" + parts.slice(1).join("/");
}
/**
* Precompile picomatch patterns for faster matching
*/
function compileMatchers(options: FilterOptions): CompiledMatchers {
// Common picomatch options
const picoOptions = {
dot: true, // Match dotfiles
windows: false, // Use forward slashes (POSIX style)
};
// For each category, create separate matchers for normal patterns and basename patterns
const inclusionMatchers = {
normal: [] as Array<(path: string) => boolean>,
basename: [] as Array<(basename: string) => boolean>,
};
const exclusionMatchers = {
normal: [] as Array<(path: string) => boolean>,
basename: [] as Array<(basename: string) => boolean>,
};
// Process inclusion patterns
if (options.pathPatterns && options.pathPatterns.length > 0) {
for (const pattern of options.pathPatterns) {
if (pattern.startsWith("*")) {
// Compile basename matchers
inclusionMatchers.basename.push(picomatch(pattern, picoOptions));
} else if (!pattern.includes("*") && !pattern.includes("?")) {
// VSCode-like behavior for non-glob patterns
inclusionMatchers.normal.push(picomatch(`${pattern}/**`, picoOptions));
} else {
// Standard pattern matching
inclusionMatchers.normal.push(picomatch(pattern, picoOptions));
}
}
}
// Process exclusion patterns
if (options.excludePathPatterns && options.excludePathPatterns.length > 0) {
for (const pattern of options.excludePathPatterns) {
if (pattern.startsWith("*")) {
// Compile basename matchers
exclusionMatchers.basename.push(picomatch(pattern, picoOptions));
} else if (!pattern.includes("*") && !pattern.includes("?")) {
// VSCode-like behavior for non-glob patterns
exclusionMatchers.normal.push(picomatch(`${pattern}/**`, picoOptions));
} else {
// Standard pattern matching
exclusionMatchers.normal.push(picomatch(pattern, picoOptions));
}
}
}
return {
inclusionMatchers,
exclusionMatchers,
hasInclusion:
inclusionMatchers.normal.length > 0 ||
inclusionMatchers.basename.length > 0,
hasExclusion:
exclusionMatchers.normal.length > 0 ||
exclusionMatchers.basename.length > 0,
};
}
// Updated CompiledMatchers interface
interface CompiledMatchers {
inclusionMatchers: {
normal: Array<(path: string) => boolean>;
basename: Array<(basename: string) => boolean>;
};
exclusionMatchers: {
normal: Array<(path: string) => boolean>;
basename: Array<(basename: string) => boolean>;
};
hasInclusion: boolean;
hasExclusion: boolean;
}
/**
* Process the TAR archive and convert it to multipart/form-data stream,
* with optimized filtering at the TAR reader level.
*/
async function processTarToMultipart(
tarStream: ReadableStream,
output: WritableStream,
filterOptions: FilterOptions,
responseOptions: ResponseOptions,
requestStartTime: number,
): Promise {
const { omitFirstSegment, rawUrlPrefix, omitBinary } = filterOptions;
const { boundary } = responseOptions;
const writer = output.getWriter();
try {
// First decompress the gzip stream
const decompressedStream = tarStream.pipeThrough(
new DecompressionStream("gzip"),
);
// Process the TAR entries
await processTarStream(decompressedStream, async (entry) => {
try {
// Skip directories
if (entry.isDirectory) {
return;
}
// Apply filtering
const filter = shouldFilter(
filterOptions,
compileMatchers(filterOptions),
entry.fileName,
entry.isDirectory,
entry.fileSize,
);
if (filter?.filter) {
if (filter?.noCallback) {
return;
}
// Process the file with filter info but without content
const processedPath = processFilePath(
entry.fileName,
omitFirstSegment,
);
// Start multipart section
await writer.write(encoder.encode(`--${boundary}\r\n`));
await writer.write(
encoder.encode(
`Content-Disposition: form-data; name="${processedPath}"; filename="${processedPath}"\r\n`,
),
);
// Add headers without content
const ext = entry.fileName.split(".").pop() || "";
const contentType = (map[ext] ||
"application/octet-stream") as string;
await writer.write(
encoder.encode(`Content-Type: ${contentType}\r\n`),
);
if (rawUrlPrefix) {
const rawUrl = `${rawUrlPrefix}${processedPath}`;
await writer.write(encoder.encode(`x-url: ${rawUrl}\r\n`));
}
const PLUGIN_NAME = "ingesttar";
await writer.write(
encoder.encode(
`x-filter: ${PLUGIN_NAME};${filter.status || "404"};${
filter.message || ""
}\r\n`,
),
);
await writer.write(
encoder.encode(`Content-Transfer-Encoding: binary\r\n\r\n\r\n`),
);
return;
}
const processedPath = processFilePath(entry.fileName, omitFirstSegment);
const ext = entry.fileName.split(".").pop() || "";
const contentType = (map[ext] || "application/octet-stream") as string;
// Start multipart section
await writer.write(encoder.encode(`--${boundary}\r\n`));
await writer.write(
encoder.encode(
`Content-Disposition: form-data; name="${processedPath}"; filename="${processedPath}"\r\n`,
),
);
// Add content type header
await writer.write(encoder.encode(`Content-Type: ${contentType}\r\n`));
// Calculate content length if available
if (entry.fileSize !== undefined) {
await writer.write(
encoder.encode(`Content-Length: ${entry.fileSize}\r\n`),
);
}
if (rawUrlPrefix) {
const rawUrl = `${rawUrlPrefix}${processedPath}`;
await writer.write(encoder.encode(`x-url: ${rawUrl}\r\n`));
}
const writeEmptyBinary = async () => {
// For binary files with rawUrlPrefix, add x-url header instead of content
await writer.write(
encoder.encode(`Content-Transfer-Encoding: binary\r\n\r\n`),
);
// Omit content for binary files when rawUrlPrefix is specified
await writer.write(encoder.encode("\r\n"));
};
if (omitBinary && binaryExtensions.includes(ext)) {
// Filter out binary files, while still responding with raw url but not with content
await writeEmptyBinary();
// Cancel the streaming if needed
await entry.fileData?.cancel();
return;
}
// Get content and hash
const { content, hash } = await getContentAndHash(entry.fileData);
if (hash) {
await writer.write(encoder.encode(`x-file-hash: ${hash}\r\n`));
}
// Determine if content is binary
const isBinaryContent = !isUtf8(content);
// Skip binary files if omitBinary is true
if (content && omitBinary && isBinaryContent) {
await writeEmptyBinary();
return;
}
// Regular handling: include content
await writer.write(
encoder.encode(
`Content-Transfer-Encoding: ${
isBinaryContent ? "binary" : "8bit"
}\r\n\r\n`,
),
);
// Write the file content
if (content) {
await writer.write(content);
}
await writer.write(encoder.encode("\r\n"));
} catch (error) {
console.error(`Error processing file ${entry.fileName}:`, error);
}
});
// End the multipart form data
await writer.write(encoder.encode(`--${boundary}--\r\n`));
const totalProcessingTime = Date.now() - requestStartTime;
console.log({ totalProcessingTime });
} catch (error) {
console.error("Error processing TAR:", error);
} finally {
await writer.close();
}
}
// TextEncoder for string to Uint8Array conversion
const encoder = new TextEncoder();
// Check if content is valid UTF-8
function isUtf8(data: Uint8Array | undefined): boolean {
if (!data) {
return false;
}
try {
const decoder = new TextDecoder("utf-8", { fatal: true, ignoreBOM: false });
decoder.decode(data);
return true;
} catch {
return false;
}
}
// Get content and generate hash
async function getContentAndHash(
stream?: ReadableStream,
): Promise<{ content?: Uint8Array; hash?: string }> {
if (!stream) {
return { content: undefined, hash: undefined };
}
const reader = stream.getReader();
const chunks: Uint8Array[] = [];
while (true) {
const { done, value } = await reader.read();
if (done) break;
chunks.push(value);
}
// Concatenate chunks
let totalLength = 0;
for (const chunk of chunks) {
totalLength += chunk.length;
}
const content = new Uint8Array(totalLength);
let offset = 0;
for (const chunk of chunks) {
content.set(chunk, offset);
offset += chunk.length;
}
// Calculate hash
const hashBuffer = await crypto.subtle.digest("SHA-256", content);
const hashArray = Array.from(new Uint8Array(hashBuffer));
const hashHex = hashArray
.map((b) => b.toString(16).padStart(2, "0"))
.join("");
return { content, hash: hashHex };
}
const shouldFilter = (
filterOptions: FilterOptions,
matchers: CompiledMatchers,
fileName: string,
isDirectory: boolean,
size?: number,
): {
filter: boolean;
noCallback?: boolean;
status?: string;
message?: string;
} => {
if (isDirectory) return { filter: true, noCallback: true }; // Skip directories
const {
omitFirstSegment,
basePath,
maxFileSize,
omitBinary,
rawUrlPrefix,
enableFuzzyMatching,
pathPatterns,
excludePathPatterns,
} = filterOptions;
// Process the path with omitFirstSegment if needed
const processedPath = omitFirstSegment
? processFilePath(fileName, true)
: fileName;
// Check maxFileSize filter
if (maxFileSize !== undefined && size !== undefined && size > maxFileSize) {
return { filter: true, status: "413", message: "Content too large" };
}
const ext = processedPath.split(".").pop()!;
if (omitBinary && !rawUrlPrefix && binaryExtensions.includes(ext)) {
// First, most efficient way, to exclude binaries
return {
filter: true,
status: "415",
message: "File has binary extension",
};
}
// Check base path filter
if (basePath && basePath.length > 0) {
const matchesBasePath = basePath.some((base) => {
// Normalize base path and filename for directory matching
const normalizedBase = surroundSlash(base);
const normalizedFilename = surroundSlash(processedPath);
return normalizedFilename.startsWith(normalizedBase);
});
if (!matchesBasePath) {
return { filter: true, status: "404", message: "No basePath matched" };
}
}
// Extract basename once for potential basename pattern matching
const basename = processedPath.split("/").pop() || "";
const normalizedPath = withoutSlash(processedPath);
// Apply inclusion patterns if defined
let included = true;
if (
matchers.hasInclusion ||
(enableFuzzyMatching && pathPatterns && pathPatterns.length > 0)
) {
// Check normal patterns from picomatch
const matchesNormalPattern = matchers.inclusionMatchers.normal.some(
(matcher) => matcher(normalizedPath),
);
// Check basename patterns from picomatch
const matchesBasenamePattern = matchers.inclusionMatchers.basename.some(
(matcher) => matcher(basename),
);
// Apply fuzzy matching directly to path patterns if enabled
const matchesFuzzyPattern =
enableFuzzyMatching && pathPatterns
? pathPatterns.some((pattern) => {
// Only apply fuzzy matching to non-glob patterns
if (!pattern.includes("*") && !pattern.includes("?")) {
return fuzzyMatch(pattern, normalizedPath);
}
return false;
})
: false;
// File is included if it matches any pattern
included =
matchesNormalPattern || matchesBasenamePattern || matchesFuzzyPattern;
}
// If not included, no need to check exclusion
if (!included) {
return {
filter: true,
status: "404",
message: "Not included in path patterns",
};
}
// Apply exclusion patterns
if (
matchers.hasExclusion ||
(enableFuzzyMatching &&
excludePathPatterns &&
excludePathPatterns.length > 0)
) {
// Check normal patterns from picomatch
const matchesNormalExcludePattern = matchers.exclusionMatchers.normal.some(
(matcher) => matcher(normalizedPath),
);
// Check basename patterns from picomatch
const matchesBasenameExcludePattern =
matchers.exclusionMatchers.basename.some((matcher) => matcher(basename));
// Apply fuzzy matching directly to exclude path patterns if enabled
const matchesFuzzyExcludePattern =
enableFuzzyMatching && excludePathPatterns
? excludePathPatterns.some((pattern) => {
// Only apply fuzzy matching to non-glob patterns
if (!pattern.includes("*") && !pattern.includes("?")) {
return fuzzyMatch(pattern, normalizedPath);
}
return false;
})
: false;
// File is excluded if it matches any exclusion pattern
const excluded =
matchesNormalExcludePattern ||
matchesBasenameExcludePattern ||
matchesFuzzyExcludePattern;
// If excluded, it takes precedence over inclusion
if (excluded) {
return {
filter: true,
status: "404",
message: "Excluded by excludePathPatterns",
};
}
}
// If we reach this point, the file should be processed
return { filter: false };
};
```
## /uithub.ingesttar/package.json
```json path="/uithub.ingesttar/package.json"
{
"dependencies": {
"binary-extensions": "^3.0.0",
"picomatch": "^4.0.2"
},
"devDependencies": {
"@types/picomatch": "^4.0.0"
}
}
```
## /uithub.ingesttar/public/default-genignore.txt
# Package manager locks and metadata
package-lock.json
*.lock
pnpm-lock.yaml
npm-shrinkwrap.json
node_modules
.pnp.*
.yarn/*
.npm
# Build artifacts and output directories
dist/
build/
out/
.output
.nuxt
.next
.cache
.parcel-cache
.webpack
.serverless
.gradle
target/
bin/
# Dependency directories
bower_components/
jspm_packages/
web_modules/
# Environment and local configuration files
.env
.env.*
.envrc
.vercel
.venv
env/
venv/
ENV/
.local
# Editor directories and files
.idea/
.vscode/
.vs/
*.swp
*.swo
*~
.DS_Store
._*
Thumbs.db
# Debug logs
npm-debug.log*
yarn-debug.log*
yarn-error.log*
lerna-debug.log*
.pnpm-debug.log*
debug.log
# Test coverage
coverage/
.nyc_output
.coverage
htmlcov/
.tox/
.pytest_cache/
__pycache__/
*.py[cod]
*$py.class
.hypothesis/
# Temporary files
*.tmp
*.temp
tmp/
temp/
.temp/
.tmp/
# IDE and tooling configuration
.eslintcache
.stylelintcache
.prettierignore
.commitlintrc
.editorconfig
.circleci/
.github/workflows/
.gitlab-ci.yml
.travis.yml
.codeclimate.yml
.dependabot/
renovate.json
# Documentation artifacts
docs/_build/
site/
.docusaurus
storybook-static/
# Minified files (often not useful for context)
*.min.js
*.min.css
# Generated assets & maps
*.map
*.d.ts
*.tsbuildinfo
# Auto-generated files
LICENSE
.gitattributes
## /uithub.ingesttar/types.ts
```ts path="/uithub.ingesttar/types.ts"
export type Env = { CREDENTIALS: string };
export interface FilterOptions {
omitFirstSegment: boolean;
omitBinary: boolean;
enableFuzzyMatching: boolean;
rawUrlPrefix: string | null;
basePath: string[];
pathPatterns: string[];
excludePathPatterns: string[];
maxFileSize: number | undefined;
genignore: boolean;
}
export interface ResponseOptions {
boundary: string;
isBrowser: boolean;
authHeader: string | null;
}
export interface RequestParams {
tarUrl: string;
filterOptions: FilterOptions;
responseOptions: ResponseOptions;
}
```
## /uithub.ingestzip/.gitignore
```gitignore path="/uithub.ingestzip/.gitignore"
node_modules
.wrangler
package-lock.json
.dev.vars
.env
```
## /uithub.outputjson/.dev.vars.example
```example path="/uithub.outputjson/.dev.vars.example"
CREDENTIALS=
```
The content has been capped at 50000 tokens. The user could consider applying other filters to refine the result. The better and more specific the context, the better the LLM can follow instructions. If the context seems verbose, the user can refine the filter using uithub. Thank you for using https://uithub.com - Perfect LLM context for any GitHub repo.