This commit is contained in:
2024-03-22 03:47:51 +05:30
parent 8bcf3d211e
commit 89819f6fe2
28440 changed files with 3211033 additions and 2 deletions

335
node_modules/cacheable-request/README.md generated vendored Normal file
View File

@@ -0,0 +1,335 @@
<h1 align="center"><br><img width="380" src="https://cacheable.org/assets/images/cacheable_symbol.svg" alt="cacheable logo"><br><br></h1>
# cacheable-request
> Wrap native HTTP requests with RFC compliant cache support
[![tests](https://github.com/jaredwray/cacheable/actions/workflows/tests.yaml/badge.svg)](https://github.com/jaredwray/cacheable/actions/workflows/tests.yaml)
[![codecov](https://codecov.io/gh/jaredwray/cacheable/branch/master/graph/badge.svg?token=LDLaqe4PsI)](https://codecov.io/gh/jaredwray/cacheable)
[![npm](https://img.shields.io/npm/dm/cacheable-request.svg)](https://www.npmjs.com/package/cacheable-request)
[![npm](https://img.shields.io/npm/v/cacheable-request.svg)](https://www.npmjs.com/package/cacheable-request)
[RFC 7234](http://httpwg.org/specs/rfc7234.html) compliant HTTP caching for native Node.js HTTP/HTTPS requests. Caching works out of the box in memory or is easily pluggable with a wide range of storage adapters.
**Note:** This is a low level wrapper around the core HTTP modules, it's not a high level request library.
# Table of Contents
* [Latest Changes](#latest-changes)
* [Features](#features)
* [Install and Usage](#install-and-usage)
* [Storage Adapters](#storage-adapters)
* [API](#api)
* [Using Hooks](#using-hooks)
* [Contributing](#contributing)
* [Ask a Question](#ask-a-question)
* [License](#license) (MIT)
# Latest Changes
## Breaking Changes with v10.0.0
This release contains breaking changes. This is the new way to use this package.
### Usage Before v10
```js
import http from 'http';
import CacheableRequest from 'cacheable-request';
// Then instead of
const req = http.request('http://example.com', cb);
req.end();
// You can do
const cacheableRequest = new CacheableRequest(http.request);
const cacheReq = cacheableRequest('http://example.com', cb);
cacheReq.on('request', req => req.end());
// Future requests to 'example.com' will be returned from cache if still valid
// You pass in any other http.request API compatible method to be wrapped with cache support:
const cacheableRequest = new CacheableRequest(https.request);
const cacheableRequest = new CacheableRequest(electron.net);
```
### Usage After v10.1.0
```js
import CacheableRequest from 'cacheable-request';
// Now You can do
const cacheableRequest = new CacheableRequest(http.request).request();
const cacheReq = cacheableRequest('http://example.com', cb);
cacheReq.on('request', req => req.end());
// Future requests to 'example.com' will be returned from cache if still valid
// You pass in any other http.request API compatible method to be wrapped with cache support:
const cacheableRequest = new CacheableRequest(https.request).request();
const cacheableRequest = new CacheableRequest(electron.net).request();
```
The biggest change is that when you do a `new` CacheableRequest you now want to call `request` method will give you the instance to use.
```diff
- const cacheableRequest = new CacheableRequest(http.request);
+ const cacheableRequest = new CacheableRequest(http.request).request();
```
### ESM Support in version 9 and higher.
We are now using pure esm support in our package. If you need to use commonjs you can use v8 or lower. To learn more about using ESM please read this from `sindresorhus`: https://gist.github.com/sindresorhus/a39789f98801d908bbc7ff3ecc99d99c
## Features
- Only stores cacheable responses as defined by RFC 7234
- Fresh cache entries are served directly from cache
- Stale cache entries are revalidated with `If-None-Match`/`If-Modified-Since` headers
- 304 responses from revalidation requests use cached body
- Updates `Age` header on cached responses
- Can completely bypass cache on a per request basis
- In memory cache by default
- Official support for Redis, Memcache, Etcd, MongoDB, SQLite, PostgreSQL and MySQL storage adapters
- Easily plug in your own or third-party storage adapters
- If DB connection fails, cache is automatically bypassed ([disabled by default](#optsautomaticfailover))
- Adds cache support to any existing HTTP code with minimal changes
- Uses [http-cache-semantics](https://github.com/pornel/http-cache-semantics) internally for HTTP RFC 7234 compliance
## Install and Usage
```shell
npm install cacheable-request
```
```js
import http from 'http';
import CacheableRequest from 'cacheable-request';
// Then instead of
const req = http.request('http://example.com', cb);
req.end();
// You can do
const cacheableRequest = new CacheableRequest(http.request).createCacheableRequest();
const cacheReq = cacheableRequest('http://example.com', cb);
cacheReq.on('request', req => req.end());
// Future requests to 'example.com' will be returned from cache if still valid
// You pass in any other http.request API compatible method to be wrapped with cache support:
const cacheableRequest = new CacheableRequest(https.request).createCacheableRequest();
const cacheableRequest = new CacheableRequest(electron.net).createCacheableRequest();
```
## Storage Adapters
`cacheable-request` uses [Keyv](https://github.com/jaredwray/keyv) to support a wide range of storage adapters.
For example, to use Redis as a cache backend, you just need to install the official Redis Keyv storage adapter:
```
npm install @keyv/redis
```
And then you can pass `CacheableRequest` your connection string:
```js
const cacheableRequest = new CacheableRequest(http.request, 'redis://user:pass@localhost:6379').createCacheableRequest();
```
[View all official Keyv storage adapters.](https://github.com/jaredwray/keyv#official-storage-adapters)
Keyv also supports anything that follows the Map API so it's easy to write your own storage adapter or use a third-party solution.
e.g The following are all valid storage adapters
```js
const storageAdapter = new Map();
// or
const storageAdapter = require('./my-storage-adapter');
// or
const QuickLRU = require('quick-lru');
const storageAdapter = new QuickLRU({ maxSize: 1000 });
const cacheableRequest = new CacheableRequest(http.request, storageAdapter).createCacheableRequest();
```
View the [Keyv docs](https://github.com/jaredwray/keyv) for more information on how to use storage adapters.
## API
### new cacheableRequest(request, [storageAdapter])
Returns the provided request function wrapped with cache support.
#### request
Type: `function`
Request function to wrap with cache support. Should be [`http.request`](https://nodejs.org/api/http.html#http_http_request_options_callback) or a similar API compatible request function.
#### storageAdapter
Type: `Keyv storage adapter`<br>
Default: `new Map()`
A [Keyv](https://github.com/jaredwray/keyv) storage adapter instance, or connection string if using with an official Keyv storage adapter.
### Instance
#### cacheableRequest(opts, [cb])
Returns an event emitter.
##### opts
Type: `object`, `string`
- Any of the default request functions options.
- Any [`http-cache-semantics`](https://github.com/kornelski/http-cache-semantics#constructor-options) options.
- Any of the following:
###### opts.cache
Type: `boolean`<br>
Default: `true`
If the cache should be used. Setting this to false will completely bypass the cache for the current request.
###### opts.strictTtl
Type: `boolean`<br>
Default: `false`
If set to `true` once a cached resource has expired it is deleted and will have to be re-requested.
If set to `false` (default), after a cached resource's TTL expires it is kept in the cache and will be revalidated on the next request with `If-None-Match`/`If-Modified-Since` headers.
###### opts.maxTtl
Type: `number`<br>
Default: `undefined`
Limits TTL. The `number` represents milliseconds.
###### opts.automaticFailover
Type: `boolean`<br>
Default: `false`
When set to `true`, if the DB connection fails we will automatically fallback to a network request. DB errors will still be emitted to notify you of the problem even though the request callback may succeed.
###### opts.forceRefresh
Type: `boolean`<br>
Default: `false`
Forces refreshing the cache. If the response could be retrieved from the cache, it will perform a new request and override the cache instead.
##### cb
Type: `function`
The callback function which will receive the response as an argument.
The response can be either a [Node.js HTTP response stream](https://nodejs.org/api/http.html#http_class_http_incomingmessage) or a [responselike object](https://github.com/lukechilds/responselike). The response will also have a `fromCache` property set with a boolean value.
##### .on('request', request)
`request` event to get the request object of the request.
**Note:** This event will only fire if an HTTP request is actually made, not when a response is retrieved from cache. However, you should always handle the `request` event to end the request and handle any potential request errors.
##### .on('response', response)
`response` event to get the response object from the HTTP request or cache.
##### .on('error', error)
`error` event emitted in case of an error with the cache.
Errors emitted here will be an instance of `CacheableRequest.RequestError` or `CacheableRequest.CacheError`. You will only ever receive a `RequestError` if the request function throws (normally caused by invalid user input). Normal request errors should be handled inside the `request` event.
To properly handle all error scenarios you should use the following pattern:
```js
cacheableRequest('example.com', cb)
.on('error', err => {
if (err instanceof CacheableRequest.CacheError) {
handleCacheError(err); // Cache error
} else if (err instanceof CacheableRequest.RequestError) {
handleRequestError(err); // Request function thrown
}
})
.on('request', req => {
req.on('error', handleRequestError); // Request error emitted
req.end();
});
```
**Note:** Database connection errors are emitted here, however `cacheable-request` will attempt to re-request the resource and bypass the cache on a connection error. Therefore a database connection error doesn't necessarily mean the request won't be fulfilled.
## Using Hooks
Hooks have been implemented since version `v9` and are very useful to modify response before saving it in cache. You can use hooks to modify response before saving it in cache.
### Add Hooks
The hook will pre compute response right before saving it in cache. You can include many hooks and it will run in order you add hook on response object.
```js
import http from 'http';
import CacheableRequest from 'cacheable-request';
const cacheableRequest = new CacheableRequest(request, cache).request();
// adding a hook to decompress response
cacheableRequest.addHook('onResponse', async (value: CacheValue, response: any) => {
const buffer = await pm(gunzip)(value.body);
value.body = buffer.toString();
return value;
});
```
here is also an example of how to add in the remote address
```js
import CacheableRequest, {CacheValue} from 'cacheable-request';
const cacheableRequest = new CacheableRequest(request, cache).request();
cacheableRequest.addHook('onResponse', (value: CacheValue, response: any) => {
if (response.connection) {
value.remoteAddress = response.connection.remoteAddress;
}
return value;
});
```
### Remove Hooks
You can also remove hook by using below
```js
CacheableRequest.removeHook('onResponse');
```
## How to Contribute
Cacheable-Request is an open source package and community driven that is maintained regularly. In addtion we have a couple of other guidelines for review:
* [CODE_OF_CONDUCT.md](CODE_OF_CONDUCT.md) - Our code of conduct
* [CONTRIBUTING.md](CONTRIBUTING.md) - How to contribute to this project
* [SECURITY.md](SECURITY.md) - Security guidelines and supported versions
## Post an Issue
To post an issue, navigate to the "Issues" tab in the main repository, and then select "New Issue." Enter a clear title describing the issue, as well as a description containing additional relevant information. Also select the label that best describes your issue type. For a bug report, for example, create an issue with the label "bug." In the description field, Be sure to include replication steps, as well as any relevant error messages.
If you're reporting a security violation, be sure to check out the project's [security policy](SECURITY.md).
Please also refer to our [Code of Conduct](CODE_OF_CONDUCT.md) for more information on how to report issues.
## Ask a Question
To ask a question, create an issue with the label "question." In the issue description, include the related code and any context that can help us answer your question.
## License
This project is under the [MIT](LICENSE) license.
MIT © Luke Childs 2017-2021 and Jared Wray 2022+

17
node_modules/cacheable-request/dist/index.d.ts generated vendored Normal file
View File

@@ -0,0 +1,17 @@
import { RequestFn, StorageAdapter, CacheResponse, CacheValue, CacheableOptions, Emitter } from './types.js';
type Func = (...args: any[]) => any;
declare class CacheableRequest {
cache: StorageAdapter;
cacheRequest: RequestFn;
hooks: Map<string, Func>;
constructor(cacheRequest: RequestFn, cacheAdapter?: StorageAdapter | string);
request: () => (options: CacheableOptions, cb?: (response: CacheResponse) => void) => Emitter;
addHook: (name: string, fn: Func) => void;
removeHook: (name: string) => boolean;
getHook: (name: string) => Func;
runHook: (name: string, ...args: any[]) => Promise<CacheValue>;
}
export default CacheableRequest;
export * from './types.js';
export declare const onResponse = "onResponse";
//# sourceMappingURL=index.d.ts.map

1
node_modules/cacheable-request/dist/index.d.ts.map generated vendored Normal file
View File

@@ -0,0 +1 @@
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAWA,OAAO,EAAC,SAAS,EAAE,cAAc,EAAE,aAAa,EAAE,UAAU,EAAE,gBAAgB,EAAuC,OAAO,EAA2B,MAAM,YAAY,CAAC;AAE1K,KAAK,IAAI,GAAG,CAAC,GAAG,IAAI,EAAE,GAAG,EAAE,KAAK,GAAG,CAAC;AAEpC,cAAM,gBAAgB;IACrB,KAAK,EAAE,cAAc,CAAC;IACtB,YAAY,EAAE,SAAS,CAAC;IACxB,KAAK,EAAE,GAAG,CAAC,MAAM,EAAE,IAAI,CAAC,CAA2B;gBACvC,YAAY,EAAE,SAAS,EAAE,YAAY,CAAC,EAAE,cAAc,GAAG,MAAM;IAmB3E,OAAO,kBAAmB,gBAAgB,kBACzB,aAAa,KAAK,IAAI,KAAG,OAAO,CAuM/C;IAEF,OAAO,SAAU,MAAM,MAAM,IAAI,UAI/B;IAEF,UAAU,SAAU,MAAM,aAA6B;IAEvD,OAAO,SAAU,MAAM,UAA0B;IAEjD,OAAO,SAAgB,MAAM,WAAW,GAAG,EAAE,KAAG,QAAQ,UAAU,CAAC,CAAoC;CACvG;AA6CD,eAAe,gBAAgB,CAAC;AAChC,cAAc,YAAY,CAAC;AAC3B,eAAO,MAAM,UAAU,eAAe,CAAC"}

275
node_modules/cacheable-request/dist/index.js generated vendored Normal file
View File

@@ -0,0 +1,275 @@
import EventEmitter from 'node:events';
import urlLib from 'node:url';
import crypto from 'node:crypto';
import stream, { PassThrough as PassThroughStream } from 'node:stream';
import normalizeUrl from 'normalize-url';
import getStream from 'get-stream';
import CachePolicy from 'http-cache-semantics';
import Response from 'responselike';
import Keyv from 'keyv';
import mimicResponse from 'mimic-response';
import { CacheError, RequestError } from './types.js';
class CacheableRequest {
constructor(cacheRequest, cacheAdapter) {
this.hooks = new Map();
this.request = () => (options, cb) => {
let url;
if (typeof options === 'string') {
url = normalizeUrlObject(urlLib.parse(options));
options = {};
}
else if (options instanceof urlLib.URL) {
url = normalizeUrlObject(urlLib.parse(options.toString()));
options = {};
}
else {
const [pathname, ...searchParts] = (options.path ?? '').split('?');
const search = searchParts.length > 0
? `?${searchParts.join('?')}`
: '';
url = normalizeUrlObject({ ...options, pathname, search });
}
options = {
headers: {},
method: 'GET',
cache: true,
strictTtl: false,
automaticFailover: false,
...options,
...urlObjectToRequestOptions(url),
};
options.headers = Object.fromEntries(entries(options.headers).map(([key, value]) => [key.toLowerCase(), value]));
const ee = new EventEmitter();
const normalizedUrlString = normalizeUrl(urlLib.format(url), {
stripWWW: false,
removeTrailingSlash: false,
stripAuthentication: false,
});
let key = `${options.method}:${normalizedUrlString}`;
// POST, PATCH, and PUT requests may be cached, depending on the response
// cache-control headers. As a result, the body of the request should be
// added to the cache key in order to avoid collisions.
if (options.body && options.method !== undefined && ['POST', 'PATCH', 'PUT'].includes(options.method)) {
if (options.body instanceof stream.Readable) {
// Streamed bodies should completely skip the cache because they may
// or may not be hashable and in either case the stream would need to
// close before the cache key could be generated.
options.cache = false;
}
else {
key += `:${crypto.createHash('md5').update(options.body).digest('hex')}`;
}
}
let revalidate = false;
let madeRequest = false;
const makeRequest = (options_) => {
madeRequest = true;
let requestErrored = false;
let requestErrorCallback = () => { };
const requestErrorPromise = new Promise(resolve => {
requestErrorCallback = () => {
if (!requestErrored) {
requestErrored = true;
resolve();
}
};
});
const handler = async (response) => {
if (revalidate) {
response.status = response.statusCode;
const revalidatedPolicy = CachePolicy.fromObject(revalidate.cachePolicy).revalidatedPolicy(options_, response);
if (!revalidatedPolicy.modified) {
response.resume();
await new Promise(resolve => {
// Skipping 'error' handler cause 'error' event should't be emitted for 304 response
response
.once('end', resolve);
});
const headers = convertHeaders(revalidatedPolicy.policy.responseHeaders());
response = new Response({ statusCode: revalidate.statusCode, headers, body: revalidate.body, url: revalidate.url });
response.cachePolicy = revalidatedPolicy.policy;
response.fromCache = true;
}
}
if (!response.fromCache) {
response.cachePolicy = new CachePolicy(options_, response, options_);
response.fromCache = false;
}
let clonedResponse;
if (options_.cache && response.cachePolicy.storable()) {
clonedResponse = cloneResponse(response);
(async () => {
try {
const bodyPromise = getStream.buffer(response);
await Promise.race([
requestErrorPromise,
new Promise(resolve => response.once('end', resolve)),
new Promise(resolve => response.once('close', resolve)), // eslint-disable-line no-promise-executor-return
]);
const body = await bodyPromise;
let value = {
url: response.url,
statusCode: response.fromCache ? revalidate.statusCode : response.statusCode,
body,
cachePolicy: response.cachePolicy.toObject(),
};
let ttl = options_.strictTtl ? response.cachePolicy.timeToLive() : undefined;
if (options_.maxTtl) {
ttl = ttl ? Math.min(ttl, options_.maxTtl) : options_.maxTtl;
}
if (this.hooks.size > 0) {
/* eslint-disable no-await-in-loop */
for (const key_ of this.hooks.keys()) {
value = await this.runHook(key_, value, response);
}
/* eslint-enable no-await-in-loop */
}
await this.cache.set(key, value, ttl);
}
catch (error) {
ee.emit('error', new CacheError(error));
}
})();
}
else if (options_.cache && revalidate) {
(async () => {
try {
await this.cache.delete(key);
}
catch (error) {
ee.emit('error', new CacheError(error));
}
})();
}
ee.emit('response', clonedResponse ?? response);
if (typeof cb === 'function') {
cb(clonedResponse ?? response);
}
};
try {
const request_ = this.cacheRequest(options_, handler);
request_.once('error', requestErrorCallback);
request_.once('abort', requestErrorCallback);
request_.once('destroy', requestErrorCallback);
ee.emit('request', request_);
}
catch (error) {
ee.emit('error', new RequestError(error));
}
};
(async () => {
const get = async (options_) => {
await Promise.resolve();
const cacheEntry = options_.cache ? await this.cache.get(key) : undefined;
if (cacheEntry === undefined && !options_.forceRefresh) {
makeRequest(options_);
return;
}
const policy = CachePolicy.fromObject(cacheEntry.cachePolicy);
if (policy.satisfiesWithoutRevalidation(options_) && !options_.forceRefresh) {
const headers = convertHeaders(policy.responseHeaders());
const response = new Response({ statusCode: cacheEntry.statusCode, headers, body: cacheEntry.body, url: cacheEntry.url });
response.cachePolicy = policy;
response.fromCache = true;
ee.emit('response', response);
if (typeof cb === 'function') {
cb(response);
}
}
else if (policy.satisfiesWithoutRevalidation(options_) && Date.now() >= policy.timeToLive() && options_.forceRefresh) {
await this.cache.delete(key);
options_.headers = policy.revalidationHeaders(options_);
makeRequest(options_);
}
else {
revalidate = cacheEntry;
options_.headers = policy.revalidationHeaders(options_);
makeRequest(options_);
}
};
const errorHandler = (error) => ee.emit('error', new CacheError(error));
if (this.cache instanceof Keyv) {
const cachek = this.cache;
cachek.once('error', errorHandler);
ee.on('error', () => cachek.removeListener('error', errorHandler));
ee.on('response', () => cachek.removeListener('error', errorHandler));
}
try {
await get(options);
}
catch (error) {
if (options.automaticFailover && !madeRequest) {
makeRequest(options);
}
ee.emit('error', new CacheError(error));
}
})();
return ee;
};
this.addHook = (name, fn) => {
if (!this.hooks.has(name)) {
this.hooks.set(name, fn);
}
};
this.removeHook = (name) => this.hooks.delete(name);
this.getHook = (name) => this.hooks.get(name);
this.runHook = async (name, ...args) => this.hooks.get(name)?.(...args);
if (cacheAdapter instanceof Keyv) {
this.cache = cacheAdapter;
}
else if (typeof cacheAdapter === 'string') {
this.cache = new Keyv({
uri: cacheAdapter,
namespace: 'cacheable-request',
});
}
else {
this.cache = new Keyv({
store: cacheAdapter,
namespace: 'cacheable-request',
});
}
this.request = this.request.bind(this);
this.cacheRequest = cacheRequest;
}
}
const entries = Object.entries;
const cloneResponse = (response) => {
const clone = new PassThroughStream({ autoDestroy: false });
mimicResponse(response, clone);
return response.pipe(clone);
};
const urlObjectToRequestOptions = (url) => {
const options = { ...url };
options.path = `${url.pathname || '/'}${url.search || ''}`;
delete options.pathname;
delete options.search;
return options;
};
const normalizeUrlObject = (url) =>
// If url was parsed by url.parse or new URL:
// - hostname will be set
// - host will be hostname[:port]
// - port will be set if it was explicit in the parsed string
// Otherwise, url was from request options:
// - hostname or host may be set
// - host shall not have port encoded
({
protocol: url.protocol,
auth: url.auth,
hostname: url.hostname || url.host || 'localhost',
port: url.port,
pathname: url.pathname,
search: url.search,
});
const convertHeaders = (headers) => {
const result = [];
for (const name of Object.keys(headers)) {
result[name.toLowerCase()] = headers[name];
}
return result;
};
export default CacheableRequest;
export * from './types.js';
export const onResponse = 'onResponse';
//# sourceMappingURL=index.js.map

1
node_modules/cacheable-request/dist/index.js.map generated vendored Normal file

File diff suppressed because one or more lines are too long

106
node_modules/cacheable-request/dist/types.d.ts generated vendored Normal file
View File

@@ -0,0 +1,106 @@
/// <reference types="node" resolution-mode="require"/>
/// <reference types="node" resolution-mode="require"/>
/// <reference types="node" resolution-mode="require"/>
/// <reference types="node" resolution-mode="require"/>
import { request, RequestOptions, ClientRequest, ServerResponse } from 'node:http';
import { URL } from 'node:url';
import { EventEmitter } from 'node:events';
import { Buffer } from 'node:buffer';
import { Store } from 'keyv';
import ResponseLike from 'responselike';
import { CachePolicyObject } from 'http-cache-semantics';
export type RequestFn = typeof request;
export type RequestFunction = typeof request;
export type CacheResponse = ServerResponse | typeof ResponseLike;
export type CacheableRequestFunction = (options: CacheableOptions, cb?: (response: CacheResponse) => void) => Emitter;
export type CacheableOptions = Options & RequestOptions | string | URL;
export type StorageAdapter = Store<any>;
export interface Options {
/**
* If the cache should be used. Setting this to `false` will completely bypass the cache for the current request.
* @default true
*/
cache?: boolean | undefined;
/**
* If set to `true` once a cached resource has expired it is deleted and will have to be re-requested.
*
* If set to `false`, after a cached resource's TTL expires it is kept in the cache and will be revalidated
* on the next request with `If-None-Match`/`If-Modified-Since` headers.
* @default false
*/
strictTtl?: boolean | undefined;
/**
* Limits TTL. The `number` represents milliseconds.
* @default undefined
*/
maxTtl?: number | undefined;
/**
* When set to `true`, if the DB connection fails we will automatically fallback to a network request.
* DB errors will still be emitted to notify you of the problem even though the request callback may succeed.
* @default false
*/
automaticFailover?: boolean | undefined;
/**
* Forces refreshing the cache. If the response could be retrieved from the cache, it will perform a
* new request and override the cache instead.
* @default false
*/
forceRefresh?: boolean | undefined;
remoteAddress?: boolean | undefined;
url?: string | undefined;
headers?: Record<string, string | string[] | undefined>;
body?: Buffer;
}
export interface CacheValue extends Record<string, any> {
url: string;
statusCode: number;
body: Buffer | string;
cachePolicy: CachePolicyObject;
}
export interface Emitter extends EventEmitter {
addListener(event: 'request', listener: (request: ClientRequest) => void): this;
addListener(event: 'response', listener: (response: CacheResponse) => void): this;
addListener(event: 'error', listener: (error: RequestError | CacheError) => void): this;
on(event: 'request', listener: (request: ClientRequest) => void): this;
on(event: 'response', listener: (response: CacheResponse) => void): this;
on(event: 'error', listener: (error: RequestError | CacheError) => void): this;
once(event: 'request', listener: (request: ClientRequest) => void): this;
once(event: 'response', listener: (response: CacheResponse) => void): this;
once(event: 'error', listener: (error: RequestError | CacheError) => void): this;
prependListener(event: 'request', listener: (request: ClientRequest) => void): this;
prependListener(event: 'response', listener: (response: CacheResponse) => void): this;
prependListener(event: 'error', listener: (error: RequestError | CacheError) => void): this;
prependOnceListener(event: 'request', listener: (request: ClientRequest) => void): this;
prependOnceListener(event: 'response', listener: (response: CacheResponse) => void): this;
prependOnceListener(event: 'error', listener: (error: RequestError | CacheError) => void): this;
removeListener(event: 'request', listener: (request: ClientRequest) => void): this;
removeListener(event: 'response', listener: (response: CacheResponse) => void): this;
removeListener(event: 'error', listener: (error: RequestError | CacheError) => void): this;
off(event: 'request', listener: (request: ClientRequest) => void): this;
off(event: 'response', listener: (response: CacheResponse) => void): this;
off(event: 'error', listener: (error: RequestError | CacheError) => void): this;
removeAllListeners(event?: 'request' | 'response' | 'error'): this;
listeners(event: 'request'): Array<(request: ClientRequest) => void>;
listeners(event: 'response'): Array<(response: CacheResponse) => void>;
listeners(event: 'error'): Array<(error: RequestError | CacheError) => void>;
rawListeners(event: 'request'): Array<(request: ClientRequest) => void>;
rawListeners(event: 'response'): Array<(response: CacheResponse) => void>;
rawListeners(event: 'error'): Array<(error: RequestError | CacheError) => void>;
emit(event: 'request', request: ClientRequest): boolean;
emit(event: 'response', response: CacheResponse): boolean;
emit(event: 'error', error: RequestError | CacheError): boolean;
eventNames(): Array<'request' | 'response' | 'error'>;
listenerCount(type: 'request' | 'response' | 'error'): number;
}
export declare class RequestError extends Error {
constructor(error: Error);
}
export declare class CacheError extends Error {
constructor(error: Error);
}
export interface UrlOption {
path: string;
pathname?: string;
search?: string;
}
//# sourceMappingURL=types.d.ts.map

1
node_modules/cacheable-request/dist/types.d.ts.map generated vendored Normal file
View File

@@ -0,0 +1 @@
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":";;;;AASA,OAAO,EAAC,OAAO,EAAE,cAAc,EAAE,aAAa,EAAE,cAAc,EAAC,MAAM,WAAW,CAAC;AACjF,OAAO,EAAC,GAAG,EAAC,MAAM,UAAU,CAAC;AAC7B,OAAO,EAAC,YAAY,EAAC,MAAM,aAAa,CAAC;AACzC,OAAO,EAAC,MAAM,EAAC,MAAM,aAAa,CAAC;AACnC,OAAO,EAAC,KAAK,EAAC,MAAM,MAAM,CAAC;AAC3B,OAAO,YAAY,MAAM,cAAc,CAAC;AACxC,OAAO,EAAC,iBAAiB,EAAC,MAAM,sBAAsB,CAAC;AAEvD,MAAM,MAAM,SAAS,GAAG,OAAO,OAAO,CAAC;AACvC,MAAM,MAAM,eAAe,GAAG,OAAO,OAAO,CAAC;AAC7C,MAAM,MAAM,aAAa,GAAG,cAAc,GAAG,OAAO,YAAY,CAAC;AAEjE,MAAM,MAAM,wBAAwB,GAAG,CACtC,OAAO,EAAE,gBAAgB,EACzB,EAAE,CAAC,EAAE,CAAC,QAAQ,EAAE,aAAa,KAAK,IAAI,KAClC,OAAO,CAAC;AAEb,MAAM,MAAM,gBAAgB,GAAG,OAAO,GAAG,cAAc,GAAG,MAAM,GAAG,GAAG,CAAC;AAEvE,MAAM,MAAM,cAAc,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC;AAExC,MAAM,WAAW,OAAO;IACvB;;;eAGK;IACL,KAAK,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IAE5B;;;;;;eAMK;IACL,SAAS,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IAEhC;;;eAGK;IACL,MAAM,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAE5B;;;;eAIK;IACL,iBAAiB,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IAExC;;;;GAIE;IACF,YAAY,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IACnC,aAAa,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IAEpC,GAAG,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAEzB,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,SAAS,CAAC,CAAC;IAExD,IAAI,CAAC,EAAE,MAAM,CAAC;CACd;AAED,MAAM,WAAW,UAAW,SAAQ,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC;IACtD,GAAG,EAAE,MAAM,CAAC;IACZ,UAAU,EAAE,MAAM,CAAC;IACnB,IAAI,EAAE,MAAM,GAAG,MAAM,CAAC;IACtB,WAAW,EAAE,iBAAiB,CAAC;CAC/B;AAED,MAAM,WAAW,OAAQ,SAAQ,YAAY;IAC5C,WAAW,CAAC,KAAK,EAAE,SAAS,EAAE,QAAQ,EAAE,CAAC,OAAO,EAAE,aAAa,KAAK,IAAI,GAAG,IAAI,CAAC;IAChF,WAAW,CACV,KAAK,EAAE,UAAU,EACjB,QAAQ,EAAE,CAAC,QAAQ,EAAE,aAAa,KAAK,IAAI,GACzC,IAAI,CAAC;IACR,WAAW,CAAC,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC,KAAK,EAAE,YAAY,GAAG,UAAU,KAAK,IAAI,GAAG,IAAI,CAAC;IACxF,EAAE,CAAC,KAAK,EAAE,SAAS,EAAE,QAAQ,EAAE,CAAC,OAAO,EAAE,aAAa,KAAK,IAAI,GAAG,IAAI,CAAC;IACvE,EAAE,CAAC,KAAK,EAAE,UAAU,EAAE,QAAQ,EAAE,CAAC,QAAQ,EAAE,aAAa,KAAK,IAAI,GAAG,IAAI,CAAC;IACzE,EAAE,CAAC,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC,KAAK,EAAE,YAAY,GAAG,UAAU,KAAK,IAAI,GAAG,IAAI,CAAC;IAC/E,IAAI,CAAC,KAAK,EAAE,SAAS,EAAE,QAAQ,EAAE,CAAC,OAAO,EAAE,aAAa,KAAK,IAAI,GAAG,IAAI,CAAC;IACzE,IAAI,CAAC,KAAK,EAAE,UAAU,EAAE,QAAQ,EAAE,CAAC,QAAQ,EAAE,aAAa,KAAK,IAAI,GAAG,IAAI,CAAC;IAC3E,IAAI,CAAC,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC,KAAK,EAAE,YAAY,GAAG,UAAU,KAAK,IAAI,GAAG,IAAI,CAAC;IACjF,eAAe,CAAC,KAAK,EAAE,SAAS,EAAE,QAAQ,EAAE,CAAC,OAAO,EAAE,aAAa,KAAK,IAAI,GAAG,IAAI,CAAC;IACpF,eAAe,CACd,KAAK,EAAE,UAAU,EACjB,QAAQ,EAAE,CAAC,QAAQ,EAAE,aAAa,KAAK,IAAI,GACzC,IAAI,CAAC;IACR,eAAe,CAAC,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC,KAAK,EAAE,YAAY,GAAG,UAAU,KAAK,IAAI,GAAG,IAAI,CAAC;IAC5F,mBAAmB,CAAC,KAAK,EAAE,SAAS,EAAE,QAAQ,EAAE,CAAC,OAAO,EAAE,aAAa,KAAK,IAAI,GAAG,IAAI,CAAC;IACxF,mBAAmB,CAClB,KAAK,EAAE,UAAU,EACjB,QAAQ,EAAE,CAAC,QAAQ,EAAE,aAAa,KAAK,IAAI,GACzC,IAAI,CAAC;IACR,mBAAmB,CAClB,KAAK,EAAE,OAAO,EACd,QAAQ,EAAE,CAAC,KAAK,EAAE,YAAY,GAAG,UAAU,KAAK,IAAI,GAClD,IAAI,CAAC;IACR,cAAc,CAAC,KAAK,EAAE,SAAS,EAAE,QAAQ,EAAE,CAAC,OAAO,EAAE,aAAa,KAAK,IAAI,GAAG,IAAI,CAAC;IACnF,cAAc,CACb,KAAK,EAAE,UAAU,EACjB,QAAQ,EAAE,CAAC,QAAQ,EAAE,aAAa,KAAK,IAAI,GACzC,IAAI,CAAC;IACR,cAAc,CAAC,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC,KAAK,EAAE,YAAY,GAAG,UAAU,KAAK,IAAI,GAAG,IAAI,CAAC;IAC3F,GAAG,CAAC,KAAK,EAAE,SAAS,EAAE,QAAQ,EAAE,CAAC,OAAO,EAAE,aAAa,KAAK,IAAI,GAAG,IAAI,CAAC;IACxE,GAAG,CAAC,KAAK,EAAE,UAAU,EAAE,QAAQ,EAAE,CAAC,QAAQ,EAAE,aAAa,KAAK,IAAI,GAAG,IAAI,CAAC;IAC1E,GAAG,CAAC,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC,KAAK,EAAE,YAAY,GAAG,UAAU,KAAK,IAAI,GAAG,IAAI,CAAC;IAChF,kBAAkB,CAAC,KAAK,CAAC,EAAE,SAAS,GAAG,UAAU,GAAG,OAAO,GAAG,IAAI,CAAC;IACnE,SAAS,CAAC,KAAK,EAAE,SAAS,GAAG,KAAK,CAAC,CAAC,OAAO,EAAE,aAAa,KAAK,IAAI,CAAC,CAAC;IACrE,SAAS,CAAC,KAAK,EAAE,UAAU,GAAG,KAAK,CAAC,CAAC,QAAQ,EAAE,aAAa,KAAK,IAAI,CAAC,CAAC;IACvE,SAAS,CAAC,KAAK,EAAE,OAAO,GAAG,KAAK,CAAC,CAAC,KAAK,EAAE,YAAY,GAAG,UAAU,KAAK,IAAI,CAAC,CAAC;IAC7E,YAAY,CAAC,KAAK,EAAE,SAAS,GAAG,KAAK,CAAC,CAAC,OAAO,EAAE,aAAa,KAAK,IAAI,CAAC,CAAC;IACxE,YAAY,CAAC,KAAK,EAAE,UAAU,GAAG,KAAK,CAAC,CAAC,QAAQ,EAAE,aAAa,KAAK,IAAI,CAAC,CAAC;IAC1E,YAAY,CAAC,KAAK,EAAE,OAAO,GAAG,KAAK,CAAC,CAAC,KAAK,EAAE,YAAY,GAAG,UAAU,KAAK,IAAI,CAAC,CAAC;IAChF,IAAI,CAAC,KAAK,EAAE,SAAS,EAAE,OAAO,EAAE,aAAa,GAAG,OAAO,CAAC;IACxD,IAAI,CAAC,KAAK,EAAE,UAAU,EAAE,QAAQ,EAAE,aAAa,GAAG,OAAO,CAAC;IAC1D,IAAI,CAAC,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,YAAY,GAAG,UAAU,GAAG,OAAO,CAAC;IAChE,UAAU,IAAI,KAAK,CAAC,SAAS,GAAG,UAAU,GAAG,OAAO,CAAC,CAAC;IACtD,aAAa,CAAC,IAAI,EAAE,SAAS,GAAG,UAAU,GAAG,OAAO,GAAG,MAAM,CAAC;CAC9D;AAED,qBAAa,YAAa,SAAQ,KAAK;gBAC1B,KAAK,EAAE,KAAK;CAIxB;AACD,qBAAa,UAAW,SAAQ,KAAK;gBACxB,KAAK,EAAE,KAAK;CAIxB;AAED,MAAM,WAAW,SAAS;IACzB,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,MAAM,CAAC,EAAE,MAAM,CAAC;CAChB"}

19
node_modules/cacheable-request/dist/types.js generated vendored Normal file
View File

@@ -0,0 +1,19 @@
// Type definitions for cacheable-request 6.0
// Project: https://github.com/lukechilds/cacheable-request#readme
// Definitions by: BendingBender <https://github.com/BendingBender>
// Paul Melnikow <https://github.com/paulmelnikow>
// Definitions: https://github.com/DefinitelyTyped/DefinitelyTyped
// TypeScript Version: 2.3
export class RequestError extends Error {
constructor(error) {
super(error.message);
Object.assign(this, error);
}
}
export class CacheError extends Error {
constructor(error) {
super(error.message);
Object.assign(this, error);
}
}
//# sourceMappingURL=types.js.map

1
node_modules/cacheable-request/dist/types.js.map generated vendored Normal file

File diff suppressed because one or more lines are too long

View File

@@ -0,0 +1,303 @@
export type Options = {
/**
@default 'http'
*/
readonly defaultProtocol?: 'https' | 'http';
/**
Prepends `defaultProtocol` to the URL if it's protocol-relative.
@default true
@example
```
normalizeUrl('//sindresorhus.com');
//=> 'http://sindresorhus.com'
normalizeUrl('//sindresorhus.com', {normalizeProtocol: false});
//=> '//sindresorhus.com'
```
*/
readonly normalizeProtocol?: boolean;
/**
Normalizes HTTPS URLs to HTTP.
@default false
@example
```
normalizeUrl('https://sindresorhus.com');
//=> 'https://sindresorhus.com'
normalizeUrl('https://sindresorhus.com', {forceHttp: true});
//=> 'http://sindresorhus.com'
```
*/
readonly forceHttp?: boolean;
/**
Normalizes HTTP URLs to HTTPS.
This option cannot be used with the `forceHttp` option at the same time.
@default false
@example
```
normalizeUrl('http://sindresorhus.com');
//=> 'http://sindresorhus.com'
normalizeUrl('http://sindresorhus.com', {forceHttps: true});
//=> 'https://sindresorhus.com'
```
*/
readonly forceHttps?: boolean;
/**
Strip the [authentication](https://en.wikipedia.org/wiki/Basic_access_authentication) part of a URL.
@default true
@example
```
normalizeUrl('https://user:password@sindresorhus.com');
//=> 'https://sindresorhus.com'
normalizeUrl('https://user:password@sindresorhus.com', {stripAuthentication: false});
//=> 'https://user:password@sindresorhus.com'
```
*/
readonly stripAuthentication?: boolean;
/**
Removes hash from the URL.
@default false
@example
```
normalizeUrl('sindresorhus.com/about.html#contact');
//=> 'http://sindresorhus.com/about.html#contact'
normalizeUrl('sindresorhus.com/about.html#contact', {stripHash: true});
//=> 'http://sindresorhus.com/about.html'
```
*/
readonly stripHash?: boolean;
/**
Remove the protocol from the URL: `http://sindresorhus.com` → `sindresorhus.com`.
It will only remove `https://` and `http://` protocols.
@default false
@example
```
normalizeUrl('https://sindresorhus.com');
//=> 'https://sindresorhus.com'
normalizeUrl('sindresorhus.com', {stripProtocol: true});
//=> 'sindresorhus.com'
```
*/
readonly stripProtocol?: boolean;
/**
Strip the [text fragment](https://web.dev/text-fragments/) part of the URL
__Note:__ The text fragment will always be removed if the `stripHash` option is set to `true`, as the hash contains the text fragment.
@default true
@example
```
normalizeUrl('http://sindresorhus.com/about.html#:~:text=hello');
//=> 'http://sindresorhus.com/about.html#'
normalizeUrl('http://sindresorhus.com/about.html#section:~:text=hello');
//=> 'http://sindresorhus.com/about.html#section'
normalizeUrl('http://sindresorhus.com/about.html#:~:text=hello', {stripTextFragment: false});
//=> 'http://sindresorhus.com/about.html#:~:text=hello'
normalizeUrl('http://sindresorhus.com/about.html#section:~:text=hello', {stripTextFragment: false});
//=> 'http://sindresorhus.com/about.html#section:~:text=hello'
```
*/
readonly stripTextFragment?: boolean;
/**
Removes `www.` from the URL.
@default true
@example
```
normalizeUrl('http://www.sindresorhus.com');
//=> 'http://sindresorhus.com'
normalizeUrl('http://www.sindresorhus.com', {stripWWW: false});
//=> 'http://www.sindresorhus.com'
```
*/
readonly stripWWW?: boolean;
/**
Removes query parameters that matches any of the provided strings or regexes.
@default [/^utm_\w+/i]
@example
```
normalizeUrl('www.sindresorhus.com?foo=bar&ref=test_ref', {
removeQueryParameters: ['ref']
});
//=> 'http://sindresorhus.com/?foo=bar'
```
If a boolean is provided, `true` will remove all the query parameters.
```
normalizeUrl('www.sindresorhus.com?foo=bar', {
removeQueryParameters: true
});
//=> 'http://sindresorhus.com'
```
`false` will not remove any query parameter.
```
normalizeUrl('www.sindresorhus.com?foo=bar&utm_medium=test&ref=test_ref', {
removeQueryParameters: false
});
//=> 'http://www.sindresorhus.com/?foo=bar&ref=test_ref&utm_medium=test'
```
*/
readonly removeQueryParameters?: ReadonlyArray<RegExp | string> | boolean;
/**
Keeps only query parameters that matches any of the provided strings or regexes.
__Note__: It overrides the `removeQueryParameters` option.
@default undefined
@example
```
normalizeUrl('https://sindresorhus.com?foo=bar&ref=unicorn', {
keepQueryParameters: ['ref']
});
//=> 'https://sindresorhus.com/?ref=unicorn'
```
*/
readonly keepQueryParameters?: ReadonlyArray<RegExp | string>;
/**
Removes trailing slash.
__Note__: Trailing slash is always removed if the URL doesn't have a pathname unless the `removeSingleSlash` option is set to `false`.
@default true
@example
```
normalizeUrl('http://sindresorhus.com/redirect/');
//=> 'http://sindresorhus.com/redirect'
normalizeUrl('http://sindresorhus.com/redirect/', {removeTrailingSlash: false});
//=> 'http://sindresorhus.com/redirect/'
normalizeUrl('http://sindresorhus.com/', {removeTrailingSlash: false});
//=> 'http://sindresorhus.com'
```
*/
readonly removeTrailingSlash?: boolean;
/**
Remove a sole `/` pathname in the output. This option is independent of `removeTrailingSlash`.
@default true
@example
```
normalizeUrl('https://sindresorhus.com/');
//=> 'https://sindresorhus.com'
normalizeUrl('https://sindresorhus.com/', {removeSingleSlash: false});
//=> 'https://sindresorhus.com/'
```
*/
readonly removeSingleSlash?: boolean;
/**
Removes the default directory index file from path that matches any of the provided strings or regexes.
When `true`, the regex `/^index\.[a-z]+$/` is used.
@default false
@example
```
normalizeUrl('www.sindresorhus.com/foo/default.php', {
removeDirectoryIndex: [/^default\.[a-z]+$/]
});
//=> 'http://sindresorhus.com/foo'
```
*/
readonly removeDirectoryIndex?: boolean | ReadonlyArray<RegExp | string>;
/**
Removes an explicit port number from the URL.
Port 443 is always removed from HTTPS URLs and 80 is always removed from HTTP URLs regardless of this option.
@default false
@example
```
normalizeUrl('sindresorhus.com:123', {
removeExplicitPort: true
});
//=> 'http://sindresorhus.com'
```
*/
readonly removeExplicitPort?: boolean;
/**
Sorts the query parameters alphabetically by key.
@default true
@example
```
normalizeUrl('www.sindresorhus.com?b=two&a=one&c=three', {
sortQueryParameters: false
});
//=> 'http://sindresorhus.com/?b=two&a=one&c=three'
```
*/
readonly sortQueryParameters?: boolean;
};
/**
[Normalize](https://en.wikipedia.org/wiki/URL_normalization) a URL.
URLs with custom protocols are not normalized and just passed through by default. Supported protocols are: `https`, `http`, `file`, and `data`.
Human-friendly URLs with basic auth (for example, `user:password@sindresorhus.com`) are not handled because basic auth conflicts with custom protocols. [Basic auth URLs are also deprecated.](https://developer.mozilla.org/en-US/docs/Web/HTTP/Authentication#access_using_credentials_in_the_url)
@param url - URL to normalize, including [data URL](https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/Data_URIs).
@example
```
import normalizeUrl from 'normalize-url';
normalizeUrl('sindresorhus.com');
//=> 'http://sindresorhus.com'
normalizeUrl('//www.sindresorhus.com:80/../baz?b=bar&a=foo');
//=> 'http://sindresorhus.com/baz?a=foo&b=bar'
```
*/
export default function normalizeUrl(url: string, options?: Options): string;

View File

@@ -0,0 +1,285 @@
// https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/Data_URIs
const DATA_URL_DEFAULT_MIME_TYPE = 'text/plain';
const DATA_URL_DEFAULT_CHARSET = 'us-ascii';
const testParameter = (name, filters) => filters.some(filter => filter instanceof RegExp ? filter.test(name) : filter === name);
const supportedProtocols = new Set([
'https:',
'http:',
'file:',
]);
const hasCustomProtocol = urlString => {
try {
const {protocol} = new URL(urlString);
return protocol.endsWith(':')
&& !protocol.includes('.')
&& !supportedProtocols.has(protocol);
} catch {
return false;
}
};
const normalizeDataURL = (urlString, {stripHash}) => {
const match = /^data:(?<type>[^,]*?),(?<data>[^#]*?)(?:#(?<hash>.*))?$/.exec(urlString);
if (!match) {
throw new Error(`Invalid URL: ${urlString}`);
}
let {type, data, hash} = match.groups;
const mediaType = type.split(';');
hash = stripHash ? '' : hash;
let isBase64 = false;
if (mediaType[mediaType.length - 1] === 'base64') {
mediaType.pop();
isBase64 = true;
}
// Lowercase MIME type
const mimeType = mediaType.shift()?.toLowerCase() ?? '';
const attributes = mediaType
.map(attribute => {
let [key, value = ''] = attribute.split('=').map(string => string.trim());
// Lowercase `charset`
if (key === 'charset') {
value = value.toLowerCase();
if (value === DATA_URL_DEFAULT_CHARSET) {
return '';
}
}
return `${key}${value ? `=${value}` : ''}`;
})
.filter(Boolean);
const normalizedMediaType = [
...attributes,
];
if (isBase64) {
normalizedMediaType.push('base64');
}
if (normalizedMediaType.length > 0 || (mimeType && mimeType !== DATA_URL_DEFAULT_MIME_TYPE)) {
normalizedMediaType.unshift(mimeType);
}
return `data:${normalizedMediaType.join(';')},${isBase64 ? data.trim() : data}${hash ? `#${hash}` : ''}`;
};
export default function normalizeUrl(urlString, options) {
options = {
defaultProtocol: 'http',
normalizeProtocol: true,
forceHttp: false,
forceHttps: false,
stripAuthentication: true,
stripHash: false,
stripTextFragment: true,
stripWWW: true,
removeQueryParameters: [/^utm_\w+/i],
removeTrailingSlash: true,
removeSingleSlash: true,
removeDirectoryIndex: false,
removeExplicitPort: false,
sortQueryParameters: true,
...options,
};
// Legacy: Append `:` to the protocol if missing.
if (typeof options.defaultProtocol === 'string' && !options.defaultProtocol.endsWith(':')) {
options.defaultProtocol = `${options.defaultProtocol}:`;
}
urlString = urlString.trim();
// Data URL
if (/^data:/i.test(urlString)) {
return normalizeDataURL(urlString, options);
}
if (hasCustomProtocol(urlString)) {
return urlString;
}
const hasRelativeProtocol = urlString.startsWith('//');
const isRelativeUrl = !hasRelativeProtocol && /^\.*\//.test(urlString);
// Prepend protocol
if (!isRelativeUrl) {
urlString = urlString.replace(/^(?!(?:\w+:)?\/\/)|^\/\//, options.defaultProtocol);
}
const urlObject = new URL(urlString);
if (options.forceHttp && options.forceHttps) {
throw new Error('The `forceHttp` and `forceHttps` options cannot be used together');
}
if (options.forceHttp && urlObject.protocol === 'https:') {
urlObject.protocol = 'http:';
}
if (options.forceHttps && urlObject.protocol === 'http:') {
urlObject.protocol = 'https:';
}
// Remove auth
if (options.stripAuthentication) {
urlObject.username = '';
urlObject.password = '';
}
// Remove hash
if (options.stripHash) {
urlObject.hash = '';
} else if (options.stripTextFragment) {
urlObject.hash = urlObject.hash.replace(/#?:~:text.*?$/i, '');
}
// Remove duplicate slashes if not preceded by a protocol
// NOTE: This could be implemented using a single negative lookbehind
// regex, but we avoid that to maintain compatibility with older js engines
// which do not have support for that feature.
if (urlObject.pathname) {
// TODO: Replace everything below with `urlObject.pathname = urlObject.pathname.replace(/(?<!\b[a-z][a-z\d+\-.]{1,50}:)\/{2,}/g, '/');` when Safari supports negative lookbehind.
// Split the string by occurrences of this protocol regex, and perform
// duplicate-slash replacement on the strings between those occurrences
// (if any).
const protocolRegex = /\b[a-z][a-z\d+\-.]{1,50}:\/\//g;
let lastIndex = 0;
let result = '';
for (;;) {
const match = protocolRegex.exec(urlObject.pathname);
if (!match) {
break;
}
const protocol = match[0];
const protocolAtIndex = match.index;
const intermediate = urlObject.pathname.slice(lastIndex, protocolAtIndex);
result += intermediate.replace(/\/{2,}/g, '/');
result += protocol;
lastIndex = protocolAtIndex + protocol.length;
}
const remnant = urlObject.pathname.slice(lastIndex, urlObject.pathname.length);
result += remnant.replace(/\/{2,}/g, '/');
urlObject.pathname = result;
}
// Decode URI octets
if (urlObject.pathname) {
try {
urlObject.pathname = decodeURI(urlObject.pathname);
} catch {}
}
// Remove directory index
if (options.removeDirectoryIndex === true) {
options.removeDirectoryIndex = [/^index\.[a-z]+$/];
}
if (Array.isArray(options.removeDirectoryIndex) && options.removeDirectoryIndex.length > 0) {
let pathComponents = urlObject.pathname.split('/');
const lastComponent = pathComponents[pathComponents.length - 1];
if (testParameter(lastComponent, options.removeDirectoryIndex)) {
pathComponents = pathComponents.slice(0, -1);
urlObject.pathname = pathComponents.slice(1).join('/') + '/';
}
}
if (urlObject.hostname) {
// Remove trailing dot
urlObject.hostname = urlObject.hostname.replace(/\.$/, '');
// Remove `www.`
if (options.stripWWW && /^www\.(?!www\.)[a-z\-\d]{1,63}\.[a-z.\-\d]{2,63}$/.test(urlObject.hostname)) {
// Each label should be max 63 at length (min: 1).
// Source: https://en.wikipedia.org/wiki/Hostname#Restrictions_on_valid_host_names
// Each TLD should be up to 63 characters long (min: 2).
// It is technically possible to have a single character TLD, but none currently exist.
urlObject.hostname = urlObject.hostname.replace(/^www\./, '');
}
}
// Remove query unwanted parameters
if (Array.isArray(options.removeQueryParameters)) {
// eslint-disable-next-line unicorn/no-useless-spread -- We are intentionally spreading to get a copy.
for (const key of [...urlObject.searchParams.keys()]) {
if (testParameter(key, options.removeQueryParameters)) {
urlObject.searchParams.delete(key);
}
}
}
if (!Array.isArray(options.keepQueryParameters) && options.removeQueryParameters === true) {
urlObject.search = '';
}
// Keep wanted query parameters
if (Array.isArray(options.keepQueryParameters) && options.keepQueryParameters.length > 0) {
// eslint-disable-next-line unicorn/no-useless-spread -- We are intentionally spreading to get a copy.
for (const key of [...urlObject.searchParams.keys()]) {
if (!testParameter(key, options.keepQueryParameters)) {
urlObject.searchParams.delete(key);
}
}
}
// Sort query parameters
if (options.sortQueryParameters) {
urlObject.searchParams.sort();
// Calling `.sort()` encodes the search parameters, so we need to decode them again.
try {
urlObject.search = decodeURIComponent(urlObject.search);
} catch {}
}
if (options.removeTrailingSlash) {
urlObject.pathname = urlObject.pathname.replace(/\/$/, '');
}
// Remove an explicit port number, excluding a default port number, if applicable
if (options.removeExplicitPort && urlObject.port) {
urlObject.port = '';
}
const oldUrlString = urlString;
// Take advantage of many of the Node `url` normalizations
urlString = urlObject.toString();
if (!options.removeSingleSlash && urlObject.pathname === '/' && !oldUrlString.endsWith('/') && urlObject.hash === '') {
urlString = urlString.replace(/\/$/, '');
}
// Remove ending `/` unless removeSingleSlash is false
if ((options.removeTrailingSlash || urlObject.pathname === '/') && urlObject.hash === '' && options.removeSingleSlash) {
urlString = urlString.replace(/\/$/, '');
}
// Restore relative protocol, if applicable
if (hasRelativeProtocol && !options.normalizeProtocol) {
urlString = urlString.replace(/^http:\/\//, '//');
}
// Remove http/https
if (options.stripProtocol) {
urlString = urlString.replace(/^(?:https?:)?\/\//, '');
}
return urlString;
}

View File

@@ -0,0 +1,9 @@
MIT License
Copyright (c) Sindre Sorhus <sindresorhus@gmail.com> (https://sindresorhus.com)
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

View File

@@ -0,0 +1,57 @@
{
"name": "normalize-url",
"version": "8.0.1",
"description": "Normalize a URL",
"license": "MIT",
"repository": "sindresorhus/normalize-url",
"funding": "https://github.com/sponsors/sindresorhus",
"author": {
"name": "Sindre Sorhus",
"email": "sindresorhus@gmail.com",
"url": "https://sindresorhus.com"
},
"type": "module",
"exports": {
"types": "./index.d.ts",
"default": "./index.js"
},
"sideEffects": false,
"engines": {
"node": ">=14.16"
},
"scripts": {
"//test": "xo && c8 ava && tsd",
"test": "c8 ava && tsd"
},
"files": [
"index.js",
"index.d.ts"
],
"keywords": [
"normalize",
"url",
"uri",
"address",
"string",
"normalization",
"normalisation",
"query",
"querystring",
"simplify",
"strip",
"trim",
"canonical"
],
"devDependencies": {
"ava": "^5.0.1",
"c8": "^7.12.0",
"tsd": "^0.24.1",
"xo": "^0.52.4"
},
"c8": {
"reporter": [
"text",
"lcov"
]
}
}

View File

@@ -0,0 +1,312 @@
# normalize-url [![Coverage Status](https://codecov.io/gh/sindresorhus/normalize-url/branch/main/graph/badge.svg)](https://codecov.io/gh/sindresorhus/normalize-url)
> [Normalize](https://en.wikipedia.org/wiki/URL_normalization) a URL
Useful when you need to display, store, deduplicate, sort, compare, etc, URLs.
**Note:** This package does **not** do URL sanitization. [Garbage in, garbage out.](https://en.wikipedia.org/wiki/Garbage_in,_garbage_out) If you use this in a server context and accept URLs as user input, it's up to you to protect against invalid URLs, [path traversal attacks](https://owasp.org/www-community/attacks/Path_Traversal), etc.
## Install
```sh
npm install normalize-url
```
## Usage
```js
import normalizeUrl from 'normalize-url';
normalizeUrl('sindresorhus.com');
//=> 'http://sindresorhus.com'
normalizeUrl('//www.sindresorhus.com:80/../baz?b=bar&a=foo');
//=> 'http://sindresorhus.com/baz?a=foo&b=bar'
```
## API
### normalizeUrl(url, options?)
URLs with custom protocols are not normalized and just passed through by default. Supported protocols are: `https`, `http`, `file`, and `data`.
Human-friendly URLs with basic auth (for example, `user:password@sindresorhus.com`) are not handled because basic auth conflicts with custom protocols. [Basic auth URLs are also deprecated.](https://developer.mozilla.org/en-US/docs/Web/HTTP/Authentication#access_using_credentials_in_the_url)
#### url
Type: `string`
URL to normalize, including [data URL](https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/Data_URIs).
#### options
Type: `object`
##### defaultProtocol
Type: `string`\
Default: `'http'`\
Values: `'https' | 'http'`
##### normalizeProtocol
Type: `boolean`\
Default: `true`
Prepend `defaultProtocol` to the URL if it's protocol-relative.
```js
normalizeUrl('//sindresorhus.com');
//=> 'http://sindresorhus.com'
normalizeUrl('//sindresorhus.com', {normalizeProtocol: false});
//=> '//sindresorhus.com'
```
##### forceHttp
Type: `boolean`\
Default: `false`
Normalize HTTPS to HTTP.
```js
normalizeUrl('https://sindresorhus.com');
//=> 'https://sindresorhus.com'
normalizeUrl('https://sindresorhus.com', {forceHttp: true});
//=> 'http://sindresorhus.com'
```
##### forceHttps
Type: `boolean`\
Default: `false`
Normalize HTTP to HTTPS.
```js
normalizeUrl('http://sindresorhus.com');
//=> 'http://sindresorhus.com'
normalizeUrl('http://sindresorhus.com', {forceHttps: true});
//=> 'https://sindresorhus.com'
```
This option cannot be used with the `forceHttp` option at the same time.
##### stripAuthentication
Type: `boolean`\
Default: `true`
Strip the [authentication](https://en.wikipedia.org/wiki/Basic_access_authentication) part of the URL.
```js
normalizeUrl('https://user:password@sindresorhus.com');
//=> 'https://sindresorhus.com'
normalizeUrl('https://user:password@sindresorhus.com', {stripAuthentication: false});
//=> 'https://user:password@sindresorhus.com'
```
##### stripHash
Type: `boolean`\
Default: `false`
Strip the hash part of the URL.
```js
normalizeUrl('sindresorhus.com/about.html#contact');
//=> 'http://sindresorhus.com/about.html#contact'
normalizeUrl('sindresorhus.com/about.html#contact', {stripHash: true});
//=> 'http://sindresorhus.com/about.html'
```
##### stripProtocol
Type: `boolean`\
Default: `false`
Remove the protocol from the URL: `http://sindresorhus.com``sindresorhus.com`.
It will only remove `https://` and `http://` protocols.
```js
normalizeUrl('https://sindresorhus.com');
//=> 'https://sindresorhus.com'
normalizeUrl('https://sindresorhus.com', {stripProtocol: true});
//=> 'sindresorhus.com'
```
##### stripTextFragment
Type: `boolean`\
Default: `true`
Strip the [text fragment](https://web.dev/text-fragments/) part of the URL.
**Note:** The text fragment will always be removed if the `stripHash` option is set to `true`, as the hash contains the text fragment.
```js
normalizeUrl('http://sindresorhus.com/about.html#:~:text=hello');
//=> 'http://sindresorhus.com/about.html#'
normalizeUrl('http://sindresorhus.com/about.html#section:~:text=hello');
//=> 'http://sindresorhus.com/about.html#section'
normalizeUrl('http://sindresorhus.com/about.html#:~:text=hello', {stripTextFragment: false});
//=> 'http://sindresorhus.com/about.html#:~:text=hello'
normalizeUrl('http://sindresorhus.com/about.html#section:~:text=hello', {stripTextFragment: false});
//=> 'http://sindresorhus.com/about.html#section:~:text=hello'
```
##### stripWWW
Type: `boolean`\
Default: `true`
Remove `www.` from the URL.
```js
normalizeUrl('http://www.sindresorhus.com');
//=> 'http://sindresorhus.com'
normalizeUrl('http://www.sindresorhus.com', {stripWWW: false});
//=> 'http://www.sindresorhus.com'
```
##### removeQueryParameters
Type: `Array<RegExp | string> | boolean`\
Default: `[/^utm_\w+/i]`
Remove query parameters that matches any of the provided strings or regexes.
```js
normalizeUrl('www.sindresorhus.com?foo=bar&ref=test_ref', {
removeQueryParameters: ['ref']
});
//=> 'http://sindresorhus.com/?foo=bar'
```
If a boolean is provided, `true` will remove all the query parameters.
```js
normalizeUrl('www.sindresorhus.com?foo=bar', {
removeQueryParameters: true
});
//=> 'http://sindresorhus.com'
```
`false` will not remove any query parameter.
```js
normalizeUrl('www.sindresorhus.com?foo=bar&utm_medium=test&ref=test_ref', {
removeQueryParameters: false
});
//=> 'http://www.sindresorhus.com/?foo=bar&ref=test_ref&utm_medium=test'
```
##### keepQueryParameters
Type: `Array<RegExp | string>`\
Default: `undefined`
Keeps only query parameters that matches any of the provided strings or regexes.
**Note:** It overrides the `removeQueryParameters` option.
```js
normalizeUrl('https://sindresorhus.com?foo=bar&ref=unicorn', {
keepQueryParameters: ['ref']
});
//=> 'https://sindresorhus.com/?ref=unicorn'
```
##### removeTrailingSlash
Type: `boolean`\
Default: `true`
Remove trailing slash.
**Note:** Trailing slash is always removed if the URL doesn't have a pathname unless the `removeSingleSlash` option is set to `false`.
```js
normalizeUrl('http://sindresorhus.com/redirect/');
//=> 'http://sindresorhus.com/redirect'
normalizeUrl('http://sindresorhus.com/redirect/', {removeTrailingSlash: false});
//=> 'http://sindresorhus.com/redirect/'
normalizeUrl('http://sindresorhus.com/', {removeTrailingSlash: false});
//=> 'http://sindresorhus.com'
```
##### removeSingleSlash
Type: `boolean`\
Default: `true`
Remove a sole `/` pathname in the output. This option is independent of `removeTrailingSlash`.
```js
normalizeUrl('https://sindresorhus.com/');
//=> 'https://sindresorhus.com'
normalizeUrl('https://sindresorhus.com/', {removeSingleSlash: false});
//=> 'https://sindresorhus.com/'
```
##### removeDirectoryIndex
Type: `boolean | Array<RegExp | string>`\
Default: `false`
Removes the default directory index file from path that matches any of the provided strings or regexes. When `true`, the regex `/^index\.[a-z]+$/` is used.
```js
normalizeUrl('www.sindresorhus.com/foo/default.php', {
removeDirectoryIndex: [/^default\.[a-z]+$/]
});
//=> 'http://sindresorhus.com/foo'
```
##### removeExplicitPort
Type: `boolean`\
Default: `false`
Removes an explicit port number from the URL.
Port 443 is always removed from HTTPS URLs and 80 is always removed from HTTP URLs regardless of this option.
```js
normalizeUrl('sindresorhus.com:123', {
removeExplicitPort: true
});
//=> 'http://sindresorhus.com'
```
##### sortQueryParameters
Type: `boolean`\
Default: `true`
Sorts the query parameters alphabetically by key.
```js
normalizeUrl('www.sindresorhus.com?b=two&a=one&c=three', {
sortQueryParameters: false
});
//=> 'http://sindresorhus.com/?b=two&a=one&c=three'
```
## Related
- [compare-urls](https://github.com/sindresorhus/compare-urls) - Compare URLs by first normalizing them

121
node_modules/cacheable-request/package.json generated vendored Normal file
View File

@@ -0,0 +1,121 @@
{
"name": "cacheable-request",
"version": "10.2.14",
"description": "Wrap native HTTP requests with RFC compliant cache support",
"license": "MIT",
"repository": "jaredwray/cacheable",
"author": "Jared Wray <me@jaredwray.com> (http://jaredwray.com)",
"type": "module",
"exports": "./dist/index.js",
"types": "./dist/index.d.ts",
"engines": {
"node": ">=14.16"
},
"scripts": {
"test": "xo && NODE_OPTIONS=--experimental-vm-modules jest --coverage ",
"prepare": "npm run build",
"build": "tsc --project tsconfig.build.json",
"clean": "rm -rf node_modules && rm -rf ./coverage && rm -rf ./test/testdb.sqlite && rm -rf ./dist"
},
"files": [
"dist"
],
"keywords": [
"HTTP",
"HTTPS",
"cache",
"caching",
"layer",
"cacheable",
"RFC 7234",
"RFC",
"7234",
"compliant"
],
"dependenciesComments": {
"@types/http-cache-semantics": "It needs to be in the dependencies list and not devDependencies because otherwise projects that use this one will be getting `Could not find a declaration file for module 'http-cache-semantics'` error when running `tsc`, see https://github.com/jaredwray/cacheable-request/issues/194 for details"
},
"dependencies": {
"@types/http-cache-semantics": "^4.0.2",
"get-stream": "^6.0.1",
"http-cache-semantics": "^4.1.1",
"keyv": "^4.5.3",
"mimic-response": "^4.0.0",
"normalize-url": "^8.0.0",
"responselike": "^3.0.0"
},
"devDependencies": {
"@keyv/sqlite": "^3.6.6",
"@types/jest": "^29.5.5",
"@types/node": "^20.8.2",
"@types/responselike": "^1.0.1",
"@types/sqlite3": "^3.1.9",
"body-parser": "^1.20.2",
"delay": "^6.0.0",
"eslint": "^8.50.0",
"eslint-plugin-jest": "^27.4.2",
"express": "^4.18.2",
"jest": "^29.7.0",
"pify": "^6.1.0",
"sqlite3": "^5.1.6",
"ts-jest": "^29.1.1",
"ts-jest-resolver": "^2.0.1",
"ts-node": "^10.9.1",
"typescript": "^5.2.2",
"xo": "^0.56.0"
},
"jest": {
"collectCoverageFrom": [
"src/**/*.{ts,js}"
],
"extensionsToTreatAsEsm": [
".ts"
],
"resolver": "ts-jest-resolver",
"moduleFileExtensions": [
"ts",
"js"
],
"transform": {
"^.+\\.(ts|tsx)$": [
"ts-jest",
{
"tsconfig": "./tsconfig.build.json",
"useESM": true
}
]
},
"testMatch": [
"**/test/*.test.(ts|js)"
],
"testEnvironment": "node"
},
"xo": {
"plugins": [
"jest"
],
"extends": [
"plugin:jest/recommended"
],
"rules": {
"@typescript-eslint/triple-slash-reference": 0,
"@typescript-eslint/no-namespace": 0,
"@typescript-eslint/no-unsafe-assignment": 0,
"@typescript-eslint/no-unsafe-call": 0,
"@typescript-eslint/ban-types": 0,
"@typescript-eslint/restrict-template-expressions": 0,
"@typescript-eslint/no-unsafe-return": 0,
"@typescript-eslint/no-unsafe-argument": 0,
"new-cap": 0,
"unicorn/no-abusive-eslint-disable": 0,
"@typescript-eslint/restrict-plus-operands": 0,
"@typescript-eslint/no-implicit-any-catch": 0,
"@typescript-eslint/consistent-type-imports": 0,
"@typescript-eslint/consistent-type-definitions": 0,
"@typescript-eslint/prefer-nullish-coalescing": 0,
"n/prefer-global/url": 0,
"n/no-deprecated-api": 0,
"unicorn/prefer-event-target": 0
}
}
}