diff options
85 files changed, 4197 insertions, 81 deletions
diff --git a/.circleci/config.yml b/.circleci/config.yml index 68d8041a2..0206c514e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -188,9 +188,7 @@ jobs: working_directory: ~/repo docker: - image: circleci/python - - image: 0xorg/ganache-cli - command: | - ganache-cli --gasLimit 10000000 --noVMErrorsOnRPCResponse --db /snapshot --noVMErrorsOnRPCResponse -p 8545 --networkId 50 -m "concert load couple harbor equip island argue ramp clarify fence smart topic" + - image: 0xorg/ganache-cli:2.2.2 - image: 0xorg/launch-kit-ci command: | yarn start:ts -p 3000:3000 @@ -202,14 +200,25 @@ jobs: key: deps9-{{ .Branch }}-{{ .Environment.CIRCLE_SHA1 }} - run: command: | + cd python-packages/json_schemas + python -m ensurepip + python -m pip install .[dev] + # HACK! installing the package should do the following + # copy for us, but it's not working in CircleCI for some + # reason. Zendesk support ticket raised (#43979) with + # CircleCI. + mkdir /usr/local/lib/python3.7/site-packages/zero_ex/json_schemas/schemas + cp -R src/zero_ex/json_schemas/schemas/* /usr/local/lib/python3.7/site-packages/zero_ex/json_schemas/schemas + - run: + command: | cd python-packages/order_utils python -m ensurepip - python -m pip install -e .[dev] + python -m pip install .[dev] - run: command: | cd python-packages/sra_client python -m ensurepip - python -m pip install -e . + python -m pip install .[dev] - save_cache: key: deps9-{{ .Branch }}-{{ .Environment.CIRCLE_SHA1 }} paths: @@ -221,6 +230,10 @@ jobs: - '.tox' - run: command: | + cd python-packages/json_schemas + coverage run setup.py test + - run: + command: | cd python-packages/order_utils coverage run setup.py test - run: @@ -228,6 +241,10 @@ jobs: cd python-packages/sra_client coverage run setup.py test - save_cache: + key: coverage-python-json-schemas-{{ .Environment.CIRCLE_SHA1 }} + paths: + - ~/repo/python-packages/json_schemas/.coverage + - save_cache: key: coverage-python-order-utils-{{ .Environment.CIRCLE_SHA1 }} paths: - ~/repo/python-packages/order_utils/.coverage @@ -249,7 +266,7 @@ jobs: command: | cd python-packages/order_utils python -m ensurepip - python -m pip install -e .[dev] + python -m pip install . - save_cache: key: deps9-{{ .Branch }}-{{ .Environment.CIRCLE_SHA1 }} paths: @@ -275,9 +292,14 @@ jobs: key: deps9-{{ .Branch }}-{{ .Environment.CIRCLE_SHA1 }} - run: command: | + cd python-packages/json_schemas + python -m ensurepip + python -m pip install .[dev] + - run: + command: | cd python-packages/order_utils python -m ensurepip - python -m pip install -e .[dev] + python -m pip install .[dev] - save_cache: key: deps9-{{ .Branch }}-{{ .Environment.CIRCLE_SHA1 }} paths: @@ -285,6 +307,10 @@ jobs: - '/usr/local/lib/python3.7/site-packages' - run: command: | + cd python-packages/json_schemas + python setup.py lint + - run: + command: | cd python-packages/order_utils python setup.py lint static-tests: @@ -357,6 +383,9 @@ jobs: - coverage-contracts-{{ .Environment.CIRCLE_SHA1 }} - restore_cache: keys: + - coverage-python-json-schemas-{{ .Environment.CIRCLE_SHA1 }} + - restore_cache: + keys: - coverage-python-order-utils-{{ .Environment.CIRCLE_SHA1 }} - run: yarn report_coverage workflows: diff --git a/.prettierignore b/.prettierignore index 7f8662b0a..d0be9ca09 100644 --- a/.prettierignore +++ b/.prettierignore @@ -20,10 +20,13 @@ lib /packages/contract-artifacts/artifacts /python-packages/order_utils/src/zero_ex/contract_artifacts/artifacts /packages/json-schemas/schemas -/python-packages/order_utils/src/zero_ex/json_schemas/schemas +/python-packages/json_schemas/src/zero_ex/json_schemas/schemas /packages/metacoin/src/contract_wrappers /packages/metacoin/artifacts /packages/sra-spec/public/ package.json scripts/postpublish_utils.js packages/sol-cov/test/fixtures/artifacts +.pytest_cache +.mypy_cache +.tox @@ -1,4 +1,4 @@ -<img src="https://github.com/0xProject/branding/blob/master/0x_Black_CMYK.png" width="200px" > +<img src="https://github.com/0xProject/branding/blob/master/0x%20Logo/PNG/0x-Logo-Black.png" width="150px" > --- @@ -24,10 +24,11 @@ Visit our [developer portal](https://0xproject.com/docs/order-utils) for a compr ### Python Packages -| Package | Version | Description | -| ------------------------------------------------ | ----------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------- | -| [`0x-order-utils`](/python-packages/order_utils) | [](https://pypi.org/project/0x-order-utils/) | A set of utilities for generating, parsing, signing and validating 0x orders | -| [`0x-sra-client`](/python-packages/sra_client) | [](https://pypi.org/project/0x-sra-client/) | A Python client for interacting with servers conforming to the Standard Relayer API specification | +| Package | Version | Description | +| -------------------------------------------------- | ------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------- | +| [`0x-json-schemas`](/python-packages/json_schemas) | [](https://pypi.org/project/0x-json-schemas/) | 0x-related JSON schemas | +| [`0x-order-utils`](/python-packages/order_utils) | [](https://pypi.org/project/0x-order-utils/) | A set of utilities for generating, parsing, signing and validating 0x orders | +| [`0x-sra-client`](/python-packages/sra_client) | [](https://pypi.org/project/0x-sra-client/) | A Python client for interacting with servers conforming to the Standard Relayer API specification | ### Typescript/Javascript Packages diff --git a/packages/asset-buyer/README.md b/packages/asset-buyer/README.md index 383a3836a..b854bda11 100644 --- a/packages/asset-buyer/README.md +++ b/packages/asset-buyer/README.md @@ -1,7 +1,5 @@ ## @0x/asset-buyer -**Warning: In Beta, has not been extensively tested.** - Convenience package for buying assets represented on the Ethereum blockchain using 0x. In its simplest form, the package helps in the usage of the [0x forwarder contract](https://github.com/0xProject/0x-protocol-specification/blob/master/v2/forwarder-specification.md), which allows users to execute [Wrapped Ether](https://weth.io/) based 0x orders without having to set allowances, wrap Ether or own ZRX, meaning they can buy tokens with Ether alone. Given some liquidity (0x signed orders), it helps estimate the Ether cost of buying a certain asset (giving a range) and then buying that asset. In its more advanced and useful form, it integrates with the [Standard Relayer API](https://github.com/0xProject/standard-relayer-api) and takes care of sourcing liquidity for you given an SRA compliant endpoint. The final result is a library that tells you what assets are available, provides an Ether based quote for any asset desired, and allows you to buy that asset using Ether alone. diff --git a/packages/dev-utils/src/web3_factory.ts b/packages/dev-utils/src/web3_factory.ts index b22bcc88b..5f8981a46 100644 --- a/packages/dev-utils/src/web3_factory.ts +++ b/packages/dev-utils/src/web3_factory.ts @@ -17,6 +17,7 @@ export interface Web3Config { shouldThrowErrorsOnGanacheRPCResponse?: boolean; // default: true rpcUrl?: string; // default: localhost:8545 shouldUseFakeGasEstimate?: boolean; // default: true + ganacheDatabasePath?: string; // default: undefined, creates a tmp dir } export const web3Factory = { @@ -45,9 +46,14 @@ export const web3Factory = { const shouldThrowErrorsOnGanacheRPCResponse = _.isUndefined(config.shouldThrowErrorsOnGanacheRPCResponse) || config.shouldThrowErrorsOnGanacheRPCResponse; + if (!_.isUndefined(config.ganacheDatabasePath)) { + // Saving the snapshot to a local db. Ganache requires this directory to exist + fs.mkdirSync(config.ganacheDatabasePath); + } provider.addProvider( new GanacheSubprovider({ vmErrorsOnRPCResponse: shouldThrowErrorsOnGanacheRPCResponse, + db_path: config.ganacheDatabasePath, gasLimit: constants.GAS_LIMIT, logger, verbose: env.parseBoolean(EnvVars.VerboseGanache), diff --git a/packages/json-schemas/schemas/asset_pairs_request_opts_schema.json b/packages/json-schemas/schemas/asset_pairs_request_opts_schema.json index 174a8fdc3..fad0bd371 100644 --- a/packages/json-schemas/schemas/asset_pairs_request_opts_schema.json +++ b/packages/json-schemas/schemas/asset_pairs_request_opts_schema.json @@ -1,5 +1,5 @@ { - "id": "/AssetPairsRequestOpts", + "id": "/AssetPairsRequestOptsSchema", "type": "object", "properties": { "assetDataA": { "$ref": "/hexSchema" }, diff --git a/packages/json-schemas/schemas/call_data_schema.json b/packages/json-schemas/schemas/call_data_schema.json index c5972e8c1..e5e6e3282 100644 --- a/packages/json-schemas/schemas/call_data_schema.json +++ b/packages/json-schemas/schemas/call_data_schema.json @@ -4,13 +4,13 @@ "from": { "$ref": "/addressSchema" }, "to": { "$ref": "/addressSchema" }, "value": { - "oneOf": [{ "$ref": "/numberSchema" }, { "$ref": "/jsNumber" }] + "oneOf": [{ "$ref": "/numberSchema" }, { "$ref": "/jsNumberSchema" }] }, "gas": { - "oneOf": [{ "$ref": "/numberSchema" }, { "$ref": "/jsNumber" }] + "oneOf": [{ "$ref": "/numberSchema" }, { "$ref": "/jsNumberSchema" }] }, "gasPrice": { - "oneOf": [{ "$ref": "/numberSchema" }, { "$ref": "/jsNumber" }] + "oneOf": [{ "$ref": "/numberSchema" }, { "$ref": "/jsNumberSchema" }] }, "data": { "type": "string", diff --git a/packages/json-schemas/schemas/ec_signature_schema.json b/packages/json-schemas/schemas/ec_signature_schema.json index bc79ca5e9..52ccfe7bb 100644 --- a/packages/json-schemas/schemas/ec_signature_schema.json +++ b/packages/json-schemas/schemas/ec_signature_schema.json @@ -1,5 +1,5 @@ { - "id": "/ECSignature", + "id": "/ecSignatureSchema", "properties": { "v": { "type": "number", diff --git a/packages/json-schemas/schemas/js_number.json b/packages/json-schemas/schemas/js_number_schema.json index 6a72d92c0..7df1c4747 100644 --- a/packages/json-schemas/schemas/js_number.json +++ b/packages/json-schemas/schemas/js_number_schema.json @@ -1,5 +1,5 @@ { - "id": "/jsNumber", + "id": "/jsNumberSchema", "type": "number", "minimum": 0 } diff --git a/packages/json-schemas/schemas/order_config_request_schema.json b/packages/json-schemas/schemas/order_config_request_schema.json index ca9b2e30e..19b043e7f 100644 --- a/packages/json-schemas/schemas/order_config_request_schema.json +++ b/packages/json-schemas/schemas/order_config_request_schema.json @@ -1,5 +1,5 @@ { - "id": "/OrderConfigRequest", + "id": "/OrderConfigRequestSchema", "type": "object", "properties": { "makerAddress": { "$ref": "/addressSchema" }, diff --git a/packages/json-schemas/schemas/orderbook_request_schema.json b/packages/json-schemas/schemas/orderbook_request_schema.json index 27848bdcb..5ce6e8ab0 100644 --- a/packages/json-schemas/schemas/orderbook_request_schema.json +++ b/packages/json-schemas/schemas/orderbook_request_schema.json @@ -1,9 +1,9 @@ { - "id": "/OrderBookRequest", + "id": "/OrderbookRequestSchema", "type": "object", "properties": { "baseAssetData": { "$ref": "/hexSchema" }, "quoteAssetData": { "$ref": "/hexSchema" } }, "required": ["baseAssetData", "quoteAssetData"] -}
\ No newline at end of file +} diff --git a/packages/json-schemas/schemas/orders_request_opts_schema.json b/packages/json-schemas/schemas/orders_request_opts_schema.json index 10da51060..4c1b9b4e9 100644 --- a/packages/json-schemas/schemas/orders_request_opts_schema.json +++ b/packages/json-schemas/schemas/orders_request_opts_schema.json @@ -1,5 +1,5 @@ { - "id": "/OrdersRequestOpts", + "id": "/OrdersRequestOptsSchema", "type": "object", "properties": { "makerAssetProxyId": { "$ref": "/hexSchema" }, diff --git a/packages/json-schemas/schemas/paged_request_opts_schema.json b/packages/json-schemas/schemas/paged_request_opts_schema.json index 7cfc73947..f143c28b0 100644 --- a/packages/json-schemas/schemas/paged_request_opts_schema.json +++ b/packages/json-schemas/schemas/paged_request_opts_schema.json @@ -1,5 +1,5 @@ { - "id": "/PagedRequestOpts", + "id": "/PagedRequestOptsSchema", "type": "object", "properties": { "page": { "type": "number" }, diff --git a/packages/json-schemas/schemas/request_opts_schema.json b/packages/json-schemas/schemas/request_opts_schema.json index b50547d18..2206f5016 100644 --- a/packages/json-schemas/schemas/request_opts_schema.json +++ b/packages/json-schemas/schemas/request_opts_schema.json @@ -1,5 +1,5 @@ { - "id": "/RequestOpts", + "id": "/RequestOptsSchema", "type": "object", "properties": { "networkId": { "type": "number" } diff --git a/packages/json-schemas/schemas/tx_data_schema.json b/packages/json-schemas/schemas/tx_data_schema.json index 4643521ce..8c3daba4e 100644 --- a/packages/json-schemas/schemas/tx_data_schema.json +++ b/packages/json-schemas/schemas/tx_data_schema.json @@ -4,13 +4,13 @@ "from": { "$ref": "/addressSchema" }, "to": { "$ref": "/addressSchema" }, "value": { - "oneOf": [{ "$ref": "/numberSchema" }, { "$ref": "/jsNumber" }] + "oneOf": [{ "$ref": "/numberSchema" }, { "$ref": "/jsNumberSchema" }] }, "gas": { - "oneOf": [{ "$ref": "/numberSchema" }, { "$ref": "/jsNumber" }] + "oneOf": [{ "$ref": "/numberSchema" }, { "$ref": "/jsNumberSchema" }] }, "gasPrice": { - "oneOf": [{ "$ref": "/numberSchema" }, { "$ref": "/jsNumber" }] + "oneOf": [{ "$ref": "/numberSchema" }, { "$ref": "/jsNumberSchema" }] }, "data": { "type": "string", diff --git a/packages/json-schemas/src/schemas.ts b/packages/json-schemas/src/schemas.ts index 050f4e625..9e8eb6959 100644 --- a/packages/json-schemas/src/schemas.ts +++ b/packages/json-schemas/src/schemas.ts @@ -8,7 +8,7 @@ import * as ecSignatureSchema from '../schemas/ec_signature_schema.json'; import * as eip712TypedDataSchema from '../schemas/eip712_typed_data_schema.json'; import * as hexSchema from '../schemas/hex_schema.json'; import * as indexFilterValuesSchema from '../schemas/index_filter_values_schema.json'; -import * as jsNumber from '../schemas/js_number.json'; +import * as jsNumber from '../schemas/js_number_schema.json'; import * as numberSchema from '../schemas/number_schema.json'; import * as orderCancellationRequestsSchema from '../schemas/order_cancel_schema.json'; import * as orderConfigRequestSchema from '../schemas/order_config_request_schema.json'; diff --git a/packages/json-schemas/tsconfig.json b/packages/json-schemas/tsconfig.json index ec573290c..7d7ce1d7e 100644 --- a/packages/json-schemas/tsconfig.json +++ b/packages/json-schemas/tsconfig.json @@ -42,7 +42,7 @@ "./schemas/relayer_api_orders_schema.json", "./schemas/signed_orders_schema.json", "./schemas/token_schema.json", - "./schemas/js_number.json", + "./schemas/js_number_schema.json", "./schemas/zero_ex_transaction_schema.json", "./schemas/tx_data_schema.json", "./schemas/index_filter_values_schema.json", diff --git a/packages/migrations/.gitignore b/packages/migrations/.gitignore new file mode 100644 index 000000000..4de81c5a8 --- /dev/null +++ b/packages/migrations/.gitignore @@ -0,0 +1,2 @@ +*.zip +0x_ganache_snapshot diff --git a/packages/migrations/Dockerfile b/packages/migrations/Dockerfile new file mode 100644 index 000000000..c4d6128c2 --- /dev/null +++ b/packages/migrations/Dockerfile @@ -0,0 +1,15 @@ +FROM mhart/alpine-node:10 + +WORKDIR /usr/src/app + +RUN npm install -g ganache-cli@6.1.6 + +ENV MNEMONIC "concert load couple harbor equip island argue ramp clarify fence smart topic" +ENV NETWORK_ID 50 +ENV VERSION "latest" +ENV SNAPSHOT_HOST "http://ganache-snapshots.0x.org.s3-website.us-east-2.amazonaws.com" +ENV SNAPSHOT_NAME "0x_ganache_snapshot" +EXPOSE 8545 + +CMD [ "sh", "-c", "wget $SNAPSHOT_HOST/$SNAPSHOT_NAME-$VERSION.zip -O snapshot.zip && unzip snapshot.zip && ganache-cli --gasLimit 10000000 --db $SNAPSHOT_NAME --noVMErrorsOnRPCResponse -p 8545 --networkId \"$NETWORK_ID\" -m \"$MNEMONIC\" -h 0.0.0.0"] + diff --git a/packages/migrations/README.md b/packages/migrations/README.md index b90d730eb..1e8b92bf8 100644 --- a/packages/migrations/README.md +++ b/packages/migrations/README.md @@ -57,3 +57,44 @@ In order to migrate the V2 0x smart contracts to TestRPC/Ganache running at `htt ```bash yarn migrate:v2 ``` + +### Publish + +#### 0x Ganache Snapshot + +The 0x Ganache snapshot can be generated and published in this package. In order to build the snapshot for this version of migrations run: + +```bash +yarn build:snapshot +``` + +This will run the migrations in Ganache and output a zip file to be uploaded to the s3 bucket. For example, after running this command you will have created `0x_ganache_snapshot-2.2.2.zip`. To publish the zip file to the s3 bucket run: + +```bash +yarn publish:snapshot +``` + +This snapshot will now be publicly available at http://ganache-snapshots.0x.org.s3.amazonaws.com/0x_ganache_snapshot-latest.zip and also versioned with the package.json version. + +#### 0x Ganache Docker Image + +We also publish a simple docker image which downloads the latest snapshot, extracts and runs Ganache. This is not required to be built when migrations change as it always downloads and runs the latest zip file. If you have made changes to the Dockerfile then a publish of the image is required. To do this run: + +```bash +yarn build:snapshot:docker +yarn publish:snapshot:docker +``` + +The result is a published docker image to the 0xorg docker registry. To start the docker image run: + +```bash +docker run -p 8545:8545 -ti 0xorg/ganache-cli:latest +``` + +This will pull the latest zip in the s3 bucket, extract and start Ganache with the snapshot. + +In the event you need a specific version of the published Ganache snapshot run the following specifying the VERSION environment variable: + +```bash +docker run -e VERSION=2.2.2 -p 8545:8545 -ti 0xorg/ganache-cli:latest +``` diff --git a/packages/migrations/package.json b/packages/migrations/package.json index 72ffe67b2..0d6ad037c 100644 --- a/packages/migrations/package.json +++ b/packages/migrations/package.json @@ -10,13 +10,22 @@ "scripts": { "build": "tsc -b", "build:ci": "yarn build", - "clean": "shx rm -rf lib", + "clean": "shx rm -rf lib ${npm_package_config_snapshot_name} ${npm_package_config_snapshot_name}-*.zip", "lint": "tslint --format stylish --project .", "migrate:v2": "run-s build script:migrate:v2", + "migrate:v2:snapshot": "run-s build script:migrate:v2:snapshot", "script:migrate:v2": "node ./lib/migrate.js", - "docs:json": "typedoc --excludePrivate --excludeExternals --target ES5 --tsconfig typedoc-tsconfig.json --json $JSON_FILE_PATH $PROJECT_FILES" + "script:migrate:v2:snapshot": "node ./lib/migrate_snapshot.js", + "docs:json": "typedoc --excludePrivate --excludeExternals --target ES5 --tsconfig typedoc-tsconfig.json --json $JSON_FILE_PATH $PROJECT_FILES", + "build:snapshot": "rm -rf ${npm_package_config_snapshot_name} && yarn migrate:v2:snapshot && zip -r \"${npm_package_config_snapshot_name}-${npm_package_version}.zip\" ${npm_package_config_snapshot_name}", + "build:snapshot:docker": "docker build --tag ${npm_package_config_docker_snapshot_name}:${npm_package_version} --tag ${npm_package_config_docker_snapshot_name}:latest .", + "publish:snapshot": "aws s3 cp ${npm_package_config_snapshot_name}-${npm_package_version}.zip ${npm_package_config_s3_snapshot_bucket} && aws s3 cp ${npm_package_config_s3_snapshot_bucket}/${npm_package_config_snapshot_name}-${npm_package_version}.zip ${npm_package_config_s3_snapshot_bucket}/${npm_package_config_snapshot_name}-latest.zip", + "publish:snapshot:docker": "docker push ${npm_package_config_docker_snapshot_name}:latest" }, "config": { + "s3_snapshot_bucket": "s3://ganache-snapshots.0x.org", + "docker_snapshot_name": "0xorg/ganache-cli", + "snapshot_name": "0x_ganache_snapshot", "postpublish": { "assets": [] } diff --git a/packages/migrations/src/migrate_snapshot.ts b/packages/migrations/src/migrate_snapshot.ts new file mode 100644 index 000000000..13fb063da --- /dev/null +++ b/packages/migrations/src/migrate_snapshot.ts @@ -0,0 +1,32 @@ +#!/usr/bin/env node +import { devConstants, web3Factory } from '@0x/dev-utils'; +import { logUtils } from '@0x/utils'; +import { Provider } from 'ethereum-types'; +import * as fs from 'fs'; +import * as _ from 'lodash'; +import * as path from 'path'; + +import { runMigrationsAsync } from './migration'; + +(async () => { + let providerConfigs; + let provider: Provider; + let txDefaults; + const packageJsonPath = path.join(__dirname, '..', 'package.json'); + const packageJsonString = fs.readFileSync(packageJsonPath, 'utf8'); + const packageJson = JSON.parse(packageJsonString); + if (_.isUndefined(packageJson.config) || _.isUndefined(packageJson.config.snapshot_name)) { + throw new Error(`Did not find 'snapshot_name' key in package.json config`); + } + + providerConfigs = { shouldUseInProcessGanache: true, ganacheDatabasePath: packageJson.config.snapshot_name }; + provider = web3Factory.getRpcProvider(providerConfigs); + txDefaults = { + from: devConstants.TESTRPC_FIRST_ADDRESS, + }; + await runMigrationsAsync(provider, txDefaults); + process.exit(0); +})().catch(err => { + logUtils.log(err); + process.exit(1); +}); diff --git a/packages/pipeline/migrations/1545440485644-CreateCopperTables.ts b/packages/pipeline/migrations/1545440485644-CreateCopperTables.ts new file mode 100644 index 000000000..64bf70af4 --- /dev/null +++ b/packages/pipeline/migrations/1545440485644-CreateCopperTables.ts @@ -0,0 +1,103 @@ +import { MigrationInterface, QueryRunner, Table } from 'typeorm'; + +const leads = new Table({ + name: 'raw.copper_leads', + columns: [ + { name: 'id', type: 'bigint', isPrimary: true }, + { name: 'name', type: 'varchar', isNullable: true }, + { name: 'first_name', type: 'varchar', isNullable: true }, + { name: 'last_name', type: 'varchar', isNullable: true }, + { name: 'middle_name', type: 'varchar', isNullable: true }, + { name: 'assignee_id', type: 'bigint', isNullable: true }, + { name: 'company_name', type: 'varchar', isNullable: true }, + { name: 'customer_source_id', type: 'bigint', isNullable: true }, + { name: 'monetary_value', type: 'integer', isNullable: true }, + { name: 'status', type: 'varchar' }, + { name: 'status_id', type: 'bigint' }, + { name: 'title', type: 'varchar', isNullable: true }, + { name: 'date_created', type: 'bigint' }, + { name: 'date_modified', type: 'bigint', isPrimary: true }, + ], +}); +const activities = new Table({ + name: 'raw.copper_activities', + columns: [ + { name: 'id', type: 'bigint', isPrimary: true }, + { name: 'parent_id', type: 'bigint' }, + { name: 'parent_type', type: 'varchar' }, + { name: 'type_id', type: 'bigint' }, + { name: 'type_category', type: 'varchar' }, + { name: 'type_name', type: 'varchar', isNullable: true }, + { name: 'user_id', type: 'bigint' }, + { name: 'old_value_id', type: 'bigint', isNullable: true }, + { name: 'old_value_name', type: 'varchar', isNullable: true }, + { name: 'new_value_id', type: 'bigint', isNullable: true }, + { name: 'new_value_name', type: 'varchar', isNullable: true }, + { name: 'date_created', type: 'bigint' }, + { name: 'date_modified', type: 'bigint', isPrimary: true }, + ], +}); + +const opportunities = new Table({ + name: 'raw.copper_opportunities', + columns: [ + { name: 'id', type: 'bigint', isPrimary: true }, + { name: 'name', type: 'varchar' }, + { name: 'assignee_id', isNullable: true, type: 'bigint' }, + { name: 'close_date', isNullable: true, type: 'varchar' }, + { name: 'company_id', isNullable: true, type: 'bigint' }, + { name: 'company_name', isNullable: true, type: 'varchar' }, + { name: 'customer_source_id', isNullable: true, type: 'bigint' }, + { name: 'loss_reason_id', isNullable: true, type: 'bigint' }, + { name: 'pipeline_id', type: 'bigint' }, + { name: 'pipeline_stage_id', type: 'bigint' }, + { name: 'primary_contact_id', isNullable: true, type: 'bigint' }, + { name: 'priority', isNullable: true, type: 'varchar' }, + { name: 'status', type: 'varchar' }, + { name: 'interaction_count', type: 'bigint' }, + { name: 'monetary_value', isNullable: true, type: 'integer' }, + { name: 'win_probability', isNullable: true, type: 'integer' }, + { name: 'date_created', type: 'bigint' }, + { name: 'date_modified', type: 'bigint', isPrimary: true }, + { name: 'custom_fields', type: 'jsonb' }, + ], +}); + +const activityTypes = new Table({ + name: 'raw.copper_activity_types', + columns: [ + { name: 'id', type: 'bigint', isPrimary: true }, + { name: 'category', type: 'varchar' }, + { name: 'name', type: 'varchar' }, + { name: 'is_disabled', type: 'boolean', isNullable: true }, + { name: 'count_as_interaction', type: 'boolean', isNullable: true }, + ], +}); + +const customFields = new Table({ + name: 'raw.copper_custom_fields', + columns: [ + { name: 'id', type: 'bigint', isPrimary: true }, + { name: 'name', type: 'varchar' }, + { name: 'data_type', type: 'varchar' }, + { name: 'field_type', type: 'varchar', isNullable: true }, + ], +}); + +export class CreateCopperTables1544055699284 implements MigrationInterface { + public async up(queryRunner: QueryRunner): Promise<any> { + await queryRunner.createTable(leads); + await queryRunner.createTable(activities); + await queryRunner.createTable(opportunities); + await queryRunner.createTable(activityTypes); + await queryRunner.createTable(customFields); + } + + public async down(queryRunner: QueryRunner): Promise<any> { + await queryRunner.dropTable(leads.name); + await queryRunner.dropTable(activities.name); + await queryRunner.dropTable(opportunities.name); + await queryRunner.dropTable(activityTypes.name); + await queryRunner.dropTable(customFields.name); + } +} diff --git a/packages/pipeline/package.json b/packages/pipeline/package.json index ab73642ec..cb3763362 100644 --- a/packages/pipeline/package.json +++ b/packages/pipeline/package.json @@ -16,7 +16,7 @@ "test:coverage": "nyc npm run test:all --all && yarn coverage:report:lcov", "coverage:report:lcov": "nyc report --reporter=text-lcov > coverage/lcov.info", "clean": "shx rm -rf lib", - "lint": "tslint --project . --format stylish --exclude ./migrations/**/*", + "lint": "tslint --project . --format stylish --exclude ./migrations/**/* --exclude ./test/fixtures/**/**/*.json", "migrate:run": "yarn typeorm migration:run --config ./lib/src/ormconfig", "migrate:revert": "yarn typeorm migration:revert --config ./lib/src/ormconfig", "migrate:create": "yarn typeorm migration:create --config ./lib/src/ormconfig --dir migrations" diff --git a/packages/pipeline/src/data_sources/copper/index.ts b/packages/pipeline/src/data_sources/copper/index.ts new file mode 100644 index 000000000..15df2fd7d --- /dev/null +++ b/packages/pipeline/src/data_sources/copper/index.ts @@ -0,0 +1,126 @@ +import { fetchAsync } from '@0x/utils'; +import Bottleneck from 'bottleneck'; + +import { + CopperActivityTypeCategory, + CopperActivityTypeResponse, + CopperCustomFieldResponse, + CopperSearchResponse, +} from '../../parsers/copper'; + +const HTTP_OK_STATUS = 200; +const COPPER_URI = 'https://api.prosperworks.com/developer_api/v1'; + +const DEFAULT_PAGINATION_PARAMS = { + page_size: 200, + sort_by: 'date_modified', + sort_direction: 'desc', +}; + +export type CopperSearchParams = CopperLeadSearchParams | CopperActivitySearchParams | CopperOpportunitySearchParams; +export interface CopperLeadSearchParams { + page_number?: number; +} + +export interface CopperActivitySearchParams { + minimum_activity_date: number; + page_number?: number; +} + +export interface CopperOpportunitySearchParams { + sort_by: string; // must override the default 'date_modified' for this endpoint + page_number?: number; +} +export enum CopperEndpoint { + Leads = '/leads/search', + Opportunities = '/opportunities/search', + Activities = '/activities/search', +} +const ONE_SECOND = 1000; + +function httpErrorCheck(response: Response): void { + if (response.status !== HTTP_OK_STATUS) { + throw new Error(`HTTP error while scraping Copper: [${JSON.stringify(response)}]`); + } +} +export class CopperSource { + private readonly _accessToken: string; + private readonly _userEmail: string; + private readonly _defaultHeaders: any; + private readonly _limiter: Bottleneck; + + constructor(maxConcurrentRequests: number, accessToken: string, userEmail: string) { + this._accessToken = accessToken; + this._userEmail = userEmail; + this._defaultHeaders = { + 'Content-Type': 'application/json', + 'X-PW-AccessToken': this._accessToken, + 'X-PW-Application': 'developer_api', + 'X-PW-UserEmail': this._userEmail, + }; + this._limiter = new Bottleneck({ + minTime: ONE_SECOND / maxConcurrentRequests, + reservoir: 30, + reservoirRefreshAmount: 30, + reservoirRefreshInterval: maxConcurrentRequests, + }); + } + + public async fetchNumberOfPagesAsync(endpoint: CopperEndpoint, searchParams?: CopperSearchParams): Promise<number> { + const resp = await this._limiter.schedule(() => + fetchAsync(COPPER_URI + endpoint, { + method: 'POST', + body: JSON.stringify({ ...DEFAULT_PAGINATION_PARAMS, ...searchParams }), + headers: this._defaultHeaders, + }), + ); + + httpErrorCheck(resp); + + // total number of records that match the request parameters + if (resp.headers.has('X-Pw-Total')) { + const totalRecords: number = parseInt(resp.headers.get('X-Pw-Total') as string, 10); // tslint:disable-line:custom-no-magic-numbers + return Math.ceil(totalRecords / DEFAULT_PAGINATION_PARAMS.page_size); + } else { + return 1; + } + } + public async fetchSearchResultsAsync<T extends CopperSearchResponse>( + endpoint: CopperEndpoint, + searchParams?: CopperSearchParams, + ): Promise<T[]> { + const request = { ...DEFAULT_PAGINATION_PARAMS, ...searchParams }; + const response = await this._limiter.schedule(() => + fetchAsync(COPPER_URI + endpoint, { + method: 'POST', + body: JSON.stringify(request), + headers: this._defaultHeaders, + }), + ); + httpErrorCheck(response); + const json: T[] = await response.json(); + return json; + } + + public async fetchActivityTypesAsync(): Promise<Map<CopperActivityTypeCategory, CopperActivityTypeResponse[]>> { + const response = await this._limiter.schedule(() => + fetchAsync(`${COPPER_URI}/activity_types`, { + method: 'GET', + headers: this._defaultHeaders, + }), + ); + httpErrorCheck(response); + return response.json(); + } + + public async fetchCustomFieldsAsync(): Promise<CopperCustomFieldResponse[]> { + const response = await this._limiter.schedule(() => + fetchAsync(`${COPPER_URI}/custom_field_definitions`, { + method: 'GET', + headers: this._defaultHeaders, + }), + ); + httpErrorCheck(response); + return response.json(); + } +} diff --git a/packages/pipeline/src/entities/copper_activity.ts b/packages/pipeline/src/entities/copper_activity.ts new file mode 100644 index 000000000..cbc034285 --- /dev/null +++ b/packages/pipeline/src/entities/copper_activity.ts @@ -0,0 +1,41 @@ +import { Column, Entity, Index, PrimaryColumn } from 'typeorm'; + +import { numberToBigIntTransformer } from '../utils'; + +@Entity({ name: 'copper_activities', schema: 'raw' }) +export class CopperActivity { + @PrimaryColumn({ type: 'bigint', transformer: numberToBigIntTransformer }) + public id!: number; + + @Index() + @Column({ name: 'parent_id', type: 'bigint', transformer: numberToBigIntTransformer }) + public parentId!: number; + @Column({ name: 'parent_type', type: 'varchar' }) + public parentType!: string; + + // join with CopperActivityType + @Index() + @Column({ name: 'type_id', type: 'bigint', transformer: numberToBigIntTransformer }) + public typeId!: number; + @Column({ name: 'type_category', type: 'varchar' }) + public typeCategory!: string; + @Column({ name: 'type_name', type: 'varchar', nullable: true }) + public typeName?: string; + + @Column({ name: 'user_id', type: 'bigint', transformer: numberToBigIntTransformer }) + public userId!: number; + @Column({ name: 'old_value_id', type: 'bigint', nullable: true, transformer: numberToBigIntTransformer }) + public oldValueId?: number; + @Column({ name: 'old_value_name', type: 'varchar', nullable: true }) + public oldValueName?: string; + @Column({ name: 'new_value_id', type: 'bigint', nullable: true, transformer: numberToBigIntTransformer }) + public newValueId?: number; + @Column({ name: 'new_value_name', type: 'varchar', nullable: true }) + public newValueName?: string; + + @Index() + @Column({ name: 'date_created', type: 'bigint', transformer: numberToBigIntTransformer }) + public dateCreated!: number; + @PrimaryColumn({ name: 'date_modified', type: 'bigint', transformer: numberToBigIntTransformer }) + public dateModified!: number; +} diff --git a/packages/pipeline/src/entities/copper_activity_type.ts b/packages/pipeline/src/entities/copper_activity_type.ts new file mode 100644 index 000000000..8fb2dcf70 --- /dev/null +++ b/packages/pipeline/src/entities/copper_activity_type.ts @@ -0,0 +1,17 @@ +import { Column, Entity, PrimaryColumn } from 'typeorm'; + +import { numberToBigIntTransformer } from '../utils'; + +@Entity({ name: 'copper_activity_types', schema: 'raw' }) +export class CopperActivityType { + @PrimaryColumn({ type: 'bigint', transformer: numberToBigIntTransformer }) + public id!: number; + @Column({ name: 'category', type: 'varchar' }) + public category!: string; + @Column({ name: 'name', type: 'varchar' }) + public name!: string; + @Column({ name: 'is_disabled', type: 'boolean', nullable: true }) + public isDisabled?: boolean; + @Column({ name: 'count_as_interaction', type: 'boolean', nullable: true }) + public countAsInteraction?: boolean; +} diff --git a/packages/pipeline/src/entities/copper_custom_field.ts b/packages/pipeline/src/entities/copper_custom_field.ts new file mode 100644 index 000000000..f23f6ab22 --- /dev/null +++ b/packages/pipeline/src/entities/copper_custom_field.ts @@ -0,0 +1,15 @@ +import { Column, Entity, PrimaryColumn } from 'typeorm'; + +import { numberToBigIntTransformer } from '../utils'; + +@Entity({ name: 'copper_custom_fields', schema: 'raw' }) +export class CopperCustomField { + @PrimaryColumn({ type: 'bigint', transformer: numberToBigIntTransformer }) + public id!: number; + @Column({ name: 'data_type', type: 'varchar' }) + public dataType!: string; + @Column({ name: 'field_type', type: 'varchar', nullable: true }) + public fieldType?: string; + @Column({ name: 'name', type: 'varchar' }) + public name!: string; +} diff --git a/packages/pipeline/src/entities/copper_lead.ts b/packages/pipeline/src/entities/copper_lead.ts new file mode 100644 index 000000000..c51ccd761 --- /dev/null +++ b/packages/pipeline/src/entities/copper_lead.ts @@ -0,0 +1,38 @@ +import { Column, Entity, Index, PrimaryColumn } from 'typeorm'; + +import { numberToBigIntTransformer } from '../utils'; + +@Entity({ name: 'copper_leads', schema: 'raw' }) +export class CopperLead { + @PrimaryColumn({ type: 'bigint', transformer: numberToBigIntTransformer }) + public id!: number; + + @Column({ name: 'name', type: 'varchar', nullable: true }) + public name?: string; + @Column({ name: 'first_name', type: 'varchar', nullable: true }) + public firstName?: string; + @Column({ name: 'last_name', type: 'varchar', nullable: true }) + public lastName?: string; + @Column({ name: 'middle_name', type: 'varchar', nullable: true }) + public middleName?: string; + @Column({ name: 'assignee_id', type: 'bigint', transformer: numberToBigIntTransformer, nullable: true }) + public assigneeId?: number; + @Column({ name: 'company_name', type: 'varchar', nullable: true }) + public companyName?: string; + @Column({ name: 'customer_source_id', type: 'bigint', transformer: numberToBigIntTransformer, nullable: true }) + public customerSourceId?: number; + @Column({ name: 'monetary_value', type: 'integer', nullable: true }) + public monetaryValue?: number; + @Column({ name: 'status', type: 'varchar' }) + public status!: string; + @Column({ name: 'status_id', type: 'bigint', transformer: numberToBigIntTransformer }) + public statusId!: number; + @Column({ name: 'title', type: 'varchar', nullable: true }) + public title?: string; + + @Index() + @Column({ name: 'date_created', type: 'bigint', transformer: numberToBigIntTransformer }) + public dateCreated!: number; + @PrimaryColumn({ name: 'date_modified', type: 'bigint', transformer: numberToBigIntTransformer }) + public dateModified!: number; +} diff --git a/packages/pipeline/src/entities/copper_opportunity.ts b/packages/pipeline/src/entities/copper_opportunity.ts new file mode 100644 index 000000000..e12bd69ce --- /dev/null +++ b/packages/pipeline/src/entities/copper_opportunity.ts @@ -0,0 +1,45 @@ +import { Column, Entity, PrimaryColumn } from 'typeorm'; + +import { numberToBigIntTransformer } from '../utils'; + +@Entity({ name: 'copper_opportunities', schema: 'raw' }) +export class CopperOpportunity { + @PrimaryColumn({ name: 'id', type: 'bigint', transformer: numberToBigIntTransformer }) + public id!: number; + @Column({ name: 'name', type: 'varchar' }) + public name!: string; + @Column({ name: 'assignee_id', nullable: true, type: 'bigint', transformer: numberToBigIntTransformer }) + public assigneeId?: number; + @Column({ name: 'close_date', nullable: true, type: 'varchar' }) + public closeDate?: string; + @Column({ name: 'company_id', nullable: true, type: 'bigint', transformer: numberToBigIntTransformer }) + public companyId?: number; + @Column({ name: 'company_name', nullable: true, type: 'varchar' }) + public companyName?: string; + @Column({ name: 'customer_source_id', nullable: true, type: 'bigint', transformer: numberToBigIntTransformer }) + public customerSourceId?: number; + @Column({ name: 'loss_reason_id', nullable: true, type: 'bigint', transformer: numberToBigIntTransformer }) + public lossReasonId?: number; + @Column({ name: 'pipeline_id', type: 'bigint', transformer: numberToBigIntTransformer }) + public pipelineId!: number; + @Column({ name: 'pipeline_stage_id', type: 'bigint', transformer: numberToBigIntTransformer }) + public pipelineStageId!: number; + @Column({ name: 'primary_contact_id', nullable: true, type: 'bigint', transformer: numberToBigIntTransformer }) + public primaryContactId?: number; + @Column({ name: 'priority', nullable: true, type: 'varchar' }) + public priority?: string; + @Column({ name: 'status', type: 'varchar' }) + public status!: string; + @Column({ name: 'interaction_count', type: 'bigint', transformer: numberToBigIntTransformer }) + public interactionCount!: number; + @Column({ name: 'monetary_value', nullable: true, type: 'integer' }) + public monetaryValue?: number; + @Column({ name: 'win_probability', nullable: true, type: 'integer' }) + public winProbability?: number; + @Column({ name: 'date_created', type: 'bigint', transformer: numberToBigIntTransformer }) + public dateCreated!: number; + @PrimaryColumn({ name: 'date_modified', type: 'bigint', transformer: numberToBigIntTransformer }) + public dateModified!: number; + @Column({ name: 'custom_fields', type: 'jsonb' }) + public customFields!: { [key: number]: number }; +} diff --git a/packages/pipeline/src/entities/index.ts b/packages/pipeline/src/entities/index.ts index cc3de78bb..27c153c07 100644 --- a/packages/pipeline/src/entities/index.ts +++ b/packages/pipeline/src/entities/index.ts @@ -16,4 +16,10 @@ export { TokenOrderbookSnapshot } from './token_order'; export { Transaction } from './transaction'; export { ERC20ApprovalEvent } from './erc20_approval_event'; +export { CopperLead } from './copper_lead'; +export { CopperActivity } from './copper_activity'; +export { CopperOpportunity } from './copper_opportunity'; +export { CopperActivityType } from './copper_activity_type'; +export { CopperCustomField } from './copper_custom_field'; + export type ExchangeEvent = ExchangeFillEvent | ExchangeCancelEvent | ExchangeCancelUpToEvent; diff --git a/packages/pipeline/src/ormconfig.ts b/packages/pipeline/src/ormconfig.ts index fe11d81d5..2700714cd 100644 --- a/packages/pipeline/src/ormconfig.ts +++ b/packages/pipeline/src/ormconfig.ts @@ -2,6 +2,11 @@ import { ConnectionOptions } from 'typeorm'; import { Block, + CopperActivity, + CopperActivityType, + CopperCustomField, + CopperLead, + CopperOpportunity, DexTrade, ERC20ApprovalEvent, ExchangeCancelEvent, @@ -18,6 +23,11 @@ import { const entities = [ Block, + CopperOpportunity, + CopperActivity, + CopperActivityType, + CopperCustomField, + CopperLead, DexTrade, ExchangeCancelEvent, ExchangeCancelUpToEvent, diff --git a/packages/pipeline/src/parsers/copper/index.ts b/packages/pipeline/src/parsers/copper/index.ts new file mode 100644 index 000000000..6c0c5abd5 --- /dev/null +++ b/packages/pipeline/src/parsers/copper/index.ts @@ -0,0 +1,259 @@ +import * as R from 'ramda'; + +import { CopperActivity, CopperActivityType, CopperCustomField, CopperLead, CopperOpportunity } from '../../entities'; + +const ONE_SECOND = 1000; +export type CopperSearchResponse = CopperLeadResponse | CopperActivityResponse | CopperOpportunityResponse; +export interface CopperLeadResponse { + id: number; + name?: string; + first_name?: string; + last_name?: string; + middle_name?: string; + assignee_id?: number; + company_name?: string; + customer_source_id?: number; + monetary_value?: number; + status: string; + status_id: number; + title?: string; + date_created: number; // in seconds + date_modified: number; // in seconds +} + +export interface CopperActivityResponse { + id: number; + parent: CopperActivityParentResponse; + type: CopperActivityTypeResponse; + user_id: number; + activity_date: number; + old_value: CopperActivityValueResponse; + new_value: CopperActivityValueResponse; + date_created: number; // in seconds + date_modified: number; // in seconds +} + +export interface CopperActivityValueResponse { + id: number; + name: string; +} +export interface CopperActivityParentResponse { + id: number; + type: string; +} + +// custom activity types +export enum CopperActivityTypeCategory { + user = 'user', + system = 'system', +} +export interface CopperActivityTypeResponse { + id: number; + category: CopperActivityTypeCategory; + name: string; + is_disabled?: boolean; + count_as_interaction?: boolean; +} + +export interface CopperOpportunityResponse { + id: number; + name: string; + assignee_id?: number; + close_date?: string; + company_id?: number; + company_name?: string; + customer_source_id?: number; + loss_reason_id?: number; + pipeline_id: number; + pipeline_stage_id: number; + primary_contact_id?: number; + priority?: string; + status: string; + tags: string[]; + interaction_count: number; + monetary_value?: number; + win_probability?: number; + date_created: number; // in seconds + date_modified: number; // in seconds + custom_fields: CopperNestedCustomFieldResponse[]; +} +interface CopperNestedCustomFieldResponse { + custom_field_definition_id: number; + value: number | number[] | null; +} +// custom fields +export enum CopperCustomFieldType { + String = 'String', + Text = 'Text', + Dropdown = 'Dropdown', + MultiSelect = 'MultiSelect', // not in API documentation but shows up in results + Date = 'Date', + Checkbox = 'Checkbox', + Float = 'Float', + URL = 'URL', + Percentage = 'Percentage', + Currency = 'Currency', + Connect = 'Connect', +} +export interface CopperCustomFieldOptionResponse { + id: number; + name: string; +} +export interface CopperCustomFieldResponse { + id: number; + name: string; + data_type: CopperCustomFieldType; + options?: CopperCustomFieldOptionResponse[]; +} +/** + * Parse response from Copper API /search/leads/ + * + * @param leads - The array of leads returned from the API + * @returns Returns an array of Copper Lead entities + */ +export function parseLeads(leads: CopperLeadResponse[]): CopperLead[] { + return leads.map(lead => { + const entity = new CopperLead(); + entity.id = lead.id; + entity.name = lead.name || undefined; + entity.firstName = lead.first_name || undefined; + entity.lastName = lead.last_name || undefined; + entity.middleName = lead.middle_name || undefined; + entity.assigneeId = lead.assignee_id || undefined; + entity.companyName = lead.company_name || undefined; + entity.customerSourceId = lead.customer_source_id || undefined; + entity.monetaryValue = lead.monetary_value || undefined; + entity.status = lead.status; + entity.statusId = lead.status_id; + entity.title = lead.title || undefined; + entity.dateCreated = lead.date_created * ONE_SECOND; + entity.dateModified = lead.date_modified * ONE_SECOND; + return entity; + }); +} + +/** + * Parse response from Copper API /search/activities/ + * + * @param activities - The array of activities returned from the API + * @returns Returns an array of Copper Activity entities + */ +export function parseActivities(activities: CopperActivityResponse[]): CopperActivity[] { + return activities.map(activity => { + const entity = new CopperActivity(); + entity.id = activity.id; + + entity.parentId = activity.parent.id; + entity.parentType = activity.parent.type; + + entity.typeId = activity.type.id; + entity.typeCategory = activity.type.category.toString(); + entity.typeName = activity.type.name; + + entity.userId = activity.user_id; + entity.dateCreated = activity.date_created * ONE_SECOND; + entity.dateModified = activity.date_modified * ONE_SECOND; + + // nested nullable fields + entity.oldValueId = R.path(['old_value', 'id'], activity); + entity.oldValueName = R.path(['old_value', 'name'], activity); + entity.newValueId = R.path(['new_value', 'id'], activity); + entity.newValueName = R.path(['new_value', 'name'], activity); + + return entity; + }); +} + +/** + * Parse response from Copper API /search/opportunities/ + * + * @param opportunities - The array of opportunities returned from the API + * @returns Returns an array of Copper Opportunity entities + */ +export function parseOpportunities(opportunities: CopperOpportunityResponse[]): CopperOpportunity[] { + return opportunities.map(opp => { + const customFields: { [key: number]: number } = opp.custom_fields + .filter(f => f.value !== null) + .map(f => ({ + ...f, + value: ([] as number[]).concat(f.value || []), // normalise all values to number[] + })) + .map(f => f.value.map(val => [f.custom_field_definition_id, val] as [number, number])) // pair each value with the custom_field_definition_id + .reduce((acc, pair) => acc.concat(pair)) // flatten + .reduce<{ [key: number]: number }>((obj, [key, value]) => { + // transform into object literal + obj[key] = value; + return obj; + }, {}); + + const entity = new CopperOpportunity(); + entity.id = opp.id; + entity.name = opp.name; + entity.assigneeId = opp.assignee_id || undefined; + entity.closeDate = opp.close_date || undefined; + entity.companyId = opp.company_id || undefined; + entity.companyName = opp.company_name || undefined; + entity.customerSourceId = opp.customer_source_id || undefined; + entity.lossReasonId = opp.loss_reason_id || undefined; + entity.pipelineId = opp.pipeline_id; + entity.pipelineStageId = opp.pipeline_stage_id; + entity.primaryContactId = opp.primary_contact_id || undefined; + entity.priority = opp.priority || undefined; + entity.status = opp.status; + entity.interactionCount = opp.interaction_count; + entity.monetaryValue = opp.monetary_value || undefined; + entity.winProbability = opp.win_probability === null ? undefined : opp.win_probability; + entity.dateCreated = opp.date_created * ONE_SECOND; + entity.dateModified = opp.date_modified * ONE_SECOND; + entity.customFields = customFields; + return entity; + }); +} + +/** + * Parse response from Copper API /activity_types/ + * + * @param activityTypeResponse - Activity Types response from the API, keyed by "user" or "system" + * @returns Returns an array of Copper Activity Type entities + */ +export function parseActivityTypes( + activityTypeResponse: Map<CopperActivityTypeCategory, CopperActivityTypeResponse[]>, +): CopperActivityType[] { + const values: CopperActivityTypeResponse[] = R.flatten(Object.values(activityTypeResponse)); + return values.map(activityType => ({ + id: activityType.id, + name: activityType.name, + category: activityType.category.toString(), + isDisabled: activityType.is_disabled, + countAsInteraction: activityType.count_as_interaction, + })); +} + +/** + * Parse response from Copper API /custom_field_definitions/ + * + * @param customFieldResponse - array of custom field definitions returned from the API, consisting of top-level fields and nested fields + * @returns Returns an array of Copper Custom Field entities + */ +export function parseCustomFields(customFieldResponse: CopperCustomFieldResponse[]): CopperCustomField[] { + function parseTopLevelField(field: CopperCustomFieldResponse): CopperCustomField[] { + const topLevelField: CopperCustomField = { + id: field.id, + name: field.name, + dataType: field.data_type.toString(), + }; + + if (field.options !== undefined) { + const nestedFields: CopperCustomField[] = field.options.map(option => ({ + id: option.id, + name: option.name, + dataType: field.name, + fieldType: 'option', + })); + return nestedFields.concat(topLevelField); + } else { + return [topLevelField]; + } + } + return R.chain(parseTopLevelField, customFieldResponse); +} diff --git a/packages/pipeline/src/scripts/pull_copper.ts b/packages/pipeline/src/scripts/pull_copper.ts new file mode 100644 index 000000000..69814f209 --- /dev/null +++ b/packages/pipeline/src/scripts/pull_copper.ts @@ -0,0 +1,129 @@ +// tslint:disable:no-console +import * as R from 'ramda'; +import { Connection, ConnectionOptions, createConnection, Repository } from 'typeorm'; + +import { CopperEndpoint, CopperSearchParams, CopperSource } from '../data_sources/copper'; +import { CopperActivity, CopperActivityType, CopperCustomField, CopperLead, CopperOpportunity } from '../entities'; +import * as ormConfig from '../ormconfig'; +import { + CopperSearchResponse, + parseActivities, + parseActivityTypes, + parseCustomFields, + parseLeads, + parseOpportunities, +} from '../parsers/copper'; +import { handleError } from '../utils'; +const ONE_SECOND = 1000; +const COPPER_RATE_LIMIT = 10; +let connection: Connection; + +(async () => { + connection = await createConnection(ormConfig as ConnectionOptions); + + const accessToken = process.env.COPPER_ACCESS_TOKEN; + const userEmail = process.env.COPPER_USER_EMAIL; + if (accessToken === undefined || userEmail === undefined) { + throw new Error('Missing required env var: COPPER_ACCESS_TOKEN and/or COPPER_USER_EMAIL'); + } + const source = new CopperSource(COPPER_RATE_LIMIT, accessToken, userEmail); + + const fetchPromises = [ + fetchAndSaveLeadsAsync(source), + fetchAndSaveOpportunitiesAsync(source), + fetchAndSaveActivitiesAsync(source), + fetchAndSaveCustomFieldsAsync(source), + fetchAndSaveActivityTypesAsync(source), + ]; + fetchPromises.forEach(async fn => { + await fn; + }); +})().catch(handleError); + +async function fetchAndSaveLeadsAsync(source: CopperSource): Promise<void> { + const repository = connection.getRepository(CopperLead); + const startTime = await getMaxAsync(connection, 'date_modified', 'raw.copper_leads'); + console.log(`Fetching Copper leads starting from ${startTime}...`); + await fetchAndSaveAsync(CopperEndpoint.Leads, source, startTime, {}, parseLeads, repository); +} + +async function fetchAndSaveOpportunitiesAsync(source: CopperSource): Promise<void> { + const repository = connection.getRepository(CopperOpportunity); + const startTime = await getMaxAsync(connection, 'date_modified', 'raw.copper_opportunities'); + console.log(`Fetching Copper opportunities starting from ${startTime}...`); + await fetchAndSaveAsync( + CopperEndpoint.Opportunities, + source, + startTime, + { sort_by: 'name' }, + parseOpportunities, + repository, + ); +} + +async function fetchAndSaveActivitiesAsync(source: CopperSource): Promise<void> { + const repository = connection.getRepository(CopperActivity); + const startTime = await getMaxAsync(connection, 'date_modified', 'raw.copper_activities'); + const searchParams = { + minimum_activity_date: Math.floor(startTime / ONE_SECOND), + }; + console.log(`Fetching Copper activities starting from ${startTime}...`); + await fetchAndSaveAsync(CopperEndpoint.Activities, source, startTime, searchParams, parseActivities, repository); +} + +async function getMaxAsync(conn: Connection, sortColumn: string, tableName: string): Promise<number> { + const queryResult = await conn.query(`SELECT MAX(${sortColumn}) as _max from ${tableName};`); + if (R.isEmpty(queryResult)) { + return 0; + } else { + return queryResult[0]._max; + } +} + +// (Xianny): Copper API doesn't allow queries to filter by date. To ensure that we are filling in ascending chronological +// order and not missing any records, we are scraping all available pages. If Copper data gets larger, +// it would make sense to search for and start filling from the first page that contains a new record. +// This search would increase our network calls and is not efficient to implement with our current small volume +// of Copper records. +async function fetchAndSaveAsync<T extends CopperSearchResponse, E>( + endpoint: CopperEndpoint, + source: CopperSource, + startTime: number, + searchParams: CopperSearchParams, + parseFn: (recs: T[]) => E[], + repository: Repository<E>, +): Promise<void> { + let saved = 0; + const numPages = await source.fetchNumberOfPagesAsync(endpoint); + try { + for (let i = numPages; i > 0; i--) { + console.log(`Fetching page ${i}/${numPages} of ${endpoint}...`); + const raw = await source.fetchSearchResultsAsync<T>(endpoint, { + ...searchParams, + page_number: i, + }); + const newRecords = raw.filter(rec => rec.date_modified * ONE_SECOND > startTime); + const parsed = parseFn(newRecords); + await repository.save<any>(parsed); + saved += newRecords.length; + } + } catch (err) { + console.log(`Error fetching ${endpoint}, stopping: ${err.stack}`); + } finally { + console.log(`Saved ${saved} items from ${endpoint}, done.`); + } +} + +async function fetchAndSaveActivityTypesAsync(source: CopperSource): Promise<void> { + console.log(`Fetching Copper activity types...`); + const activityTypes = await source.fetchActivityTypesAsync(); + const repository = connection.getRepository(CopperActivityType); + await repository.save(parseActivityTypes(activityTypes)); +} + +async function fetchAndSaveCustomFieldsAsync(source: CopperSource): Promise<void> { + console.log(`Fetching Copper custom fields...`); + const customFields = await source.fetchCustomFieldsAsync(); + const repository = connection.getRepository(CopperCustomField); + await repository.save(parseCustomFields(customFields)); +} diff --git a/packages/pipeline/src/utils/transformers/number_to_bigint.ts b/packages/pipeline/src/utils/transformers/number_to_bigint.ts index 85560c1f0..9736d7c18 100644 --- a/packages/pipeline/src/utils/transformers/number_to_bigint.ts +++ b/packages/pipeline/src/utils/transformers/number_to_bigint.ts @@ -9,8 +9,12 @@ const decimalRadix = 10; // https://github.com/typeorm/typeorm/issues/2400 for more information. export class NumberToBigIntTransformer implements ValueTransformer { // tslint:disable-next-line:prefer-function-over-method - public to(value: number): string { - return value.toString(); + public to(value: number): string | null { + if (value === null || value === undefined) { + return null; + } else { + return value.toString(); + } } // tslint:disable-next-line:prefer-function-over-method diff --git a/packages/pipeline/test/entities/copper_test.ts b/packages/pipeline/test/entities/copper_test.ts new file mode 100644 index 000000000..2543364e6 --- /dev/null +++ b/packages/pipeline/test/entities/copper_test.ts @@ -0,0 +1,54 @@ +import 'mocha'; +import 'reflect-metadata'; + +import { + CopperActivity, + CopperActivityType, + CopperCustomField, + CopperLead, + CopperOpportunity, +} from '../../src/entities'; +import { createDbConnectionOnceAsync } from '../db_setup'; +import { + ParsedActivities, + ParsedActivityTypes, + ParsedCustomFields, + ParsedLeads, + ParsedOpportunities, +} from '../fixtures/copper/parsed_entities'; +import { chaiSetup } from '../utils/chai_setup'; + +import { testSaveAndFindEntityAsync } from './util'; + +chaiSetup.configure(); + +describe('Copper entities', () => { + describe('save and find', async () => { + it('Copper lead', async () => { + const connection = await createDbConnectionOnceAsync(); + const repository = connection.getRepository(CopperLead); + ParsedLeads.forEach(async entity => testSaveAndFindEntityAsync(repository, entity)); + }); + it('Copper activity', async () => { + const connection = await createDbConnectionOnceAsync(); + const repository = connection.getRepository(CopperActivity); + ParsedActivities.forEach(async entity => testSaveAndFindEntityAsync(repository, entity)); + }); + // searching on jsonb fields is broken in typeorm + it.skip('Copper opportunity', async () => { + const connection = await createDbConnectionOnceAsync(); + const repository = connection.getRepository(CopperOpportunity); + ParsedOpportunities.forEach(async entity => testSaveAndFindEntityAsync(repository, entity)); + }); + it('Copper activity type', async () => { + const connection = await createDbConnectionOnceAsync(); + const repository = connection.getRepository(CopperActivityType); + ParsedActivityTypes.forEach(async entity => testSaveAndFindEntityAsync(repository, entity)); + }); + it('Copper custom field', async () => { + const connection = await createDbConnectionOnceAsync(); + const repository = connection.getRepository(CopperCustomField); + ParsedCustomFields.forEach(async entity => testSaveAndFindEntityAsync(repository, entity)); + }); + }); +}); diff --git a/packages/pipeline/test/entities/util.ts b/packages/pipeline/test/entities/util.ts index 043a3b15d..42df23a4a 100644 --- a/packages/pipeline/test/entities/util.ts +++ b/packages/pipeline/test/entities/util.ts @@ -15,9 +15,9 @@ const expect = chai.expect; * @param entity An instance of a TypeORM entity which will be saved/retrieved from the database. */ export async function testSaveAndFindEntityAsync<T>(repository: Repository<T>, entity: T): Promise<void> { - // Note(albrow): We are forced to use an 'as any' hack here because + // Note(albrow): We are forced to use an 'any' hack here because // TypeScript complains about stack depth when checking the types. - await repository.save(entity as any); + await repository.save<any>(entity); const gotEntity = await repository.findOneOrFail({ where: entity, }); diff --git a/packages/pipeline/test/fixtures/copper/api_v1_activity_types.json b/packages/pipeline/test/fixtures/copper/api_v1_activity_types.json new file mode 100644 index 000000000..dbd39c31b --- /dev/null +++ b/packages/pipeline/test/fixtures/copper/api_v1_activity_types.json @@ -0,0 +1,24 @@ +{ + "user": [ + { "id": 0, "category": "user", "name": "Note", "is_disabled": false, "count_as_interaction": false }, + { "id": 660496, "category": "user", "name": "To Do", "is_disabled": false, "count_as_interaction": false }, + { "id": 660495, "category": "user", "name": "Meeting", "is_disabled": false, "count_as_interaction": true }, + { "id": 660494, "category": "user", "name": "Phone Call", "is_disabled": false, "count_as_interaction": true } + ], + "system": [ + { + "id": 1, + "category": "system", + "name": "Property Changed", + "is_disabled": false, + "count_as_interaction": false + }, + { + "id": 3, + "category": "system", + "name": "Pipeline Stage Changed", + "is_disabled": false, + "count_as_interaction": false + } + ] +} diff --git a/packages/pipeline/test/fixtures/copper/api_v1_activity_types.ts b/packages/pipeline/test/fixtures/copper/api_v1_activity_types.ts new file mode 100644 index 000000000..fd2d62a6c --- /dev/null +++ b/packages/pipeline/test/fixtures/copper/api_v1_activity_types.ts @@ -0,0 +1,16 @@ +import { CopperActivityType } from '../../../src/entities'; +const ParsedActivityTypes: CopperActivityType[] = [ + { id: 0, name: 'Note', category: 'user', isDisabled: false, countAsInteraction: false }, + { id: 660496, name: 'To Do', category: 'user', isDisabled: false, countAsInteraction: false }, + { id: 660495, name: 'Meeting', category: 'user', isDisabled: false, countAsInteraction: true }, + { id: 660494, name: 'Phone Call', category: 'user', isDisabled: false, countAsInteraction: true }, + { id: 1, name: 'Property Changed', category: 'system', isDisabled: false, countAsInteraction: false }, + { + id: 3, + name: 'Pipeline Stage Changed', + category: 'system', + isDisabled: false, + countAsInteraction: false, + }, +]; +export { ParsedActivityTypes }; diff --git a/packages/pipeline/test/fixtures/copper/api_v1_custom_field_definitions.json b/packages/pipeline/test/fixtures/copper/api_v1_custom_field_definitions.json new file mode 100644 index 000000000..c6665cb0f --- /dev/null +++ b/packages/pipeline/test/fixtures/copper/api_v1_custom_field_definitions.json @@ -0,0 +1,38 @@ +[ + { + "id": 261066, + "name": "Integration Type", + "canonical_name": null, + "data_type": "MultiSelect", + "available_on": ["opportunity", "company", "person"], + "options": [ + { "id": 394020, "name": "Strategic Relationship", "rank": 7 }, + { "id": 394013, "name": "ERC-20 Exchange", "rank": 0 }, + { "id": 394014, "name": "ERC-721 Marketplace", "rank": 1 }, + { "id": 394015, "name": "Trade Widget", "rank": 2 }, + { "id": 394016, "name": "Prediction Market Exchange", "rank": 3 }, + { "id": 394017, "name": "Security Token Exchange", "rank": 4 }, + { "id": 394018, "name": "Complementary Company", "rank": 5 }, + { "id": 394019, "name": "Service Provider", "rank": 6 } + ] + }, + { + "id": 261067, + "name": "Company Type", + "canonical_name": null, + "data_type": "Dropdown", + "available_on": ["company", "opportunity", "person"], + "options": [ + { "id": 394129, "name": "Market Maker", "rank": 6 }, + { "id": 394130, "name": "Events", "rank": 2 }, + { "id": 394023, "name": "Exchange", "rank": 3 }, + { "id": 394024, "name": "Investor", "rank": 5 }, + { "id": 394026, "name": "Service Provider", "rank": 8 }, + { "id": 394027, "name": "Wallet", "rank": 9 }, + { "id": 394134, "name": "Game", "rank": 4 }, + { "id": 394025, "name": "OTC", "rank": 7 }, + { "id": 394021, "name": "Blockchain/Protocol", "rank": 0 }, + { "id": 394022, "name": "dApp", "rank": 1 } + ] + } +] diff --git a/packages/pipeline/test/fixtures/copper/api_v1_custom_field_definitions.ts b/packages/pipeline/test/fixtures/copper/api_v1_custom_field_definitions.ts new file mode 100644 index 000000000..a44bbd2c3 --- /dev/null +++ b/packages/pipeline/test/fixtures/copper/api_v1_custom_field_definitions.ts @@ -0,0 +1,39 @@ +import { CopperCustomField } from '../../../src/entities'; +const ParsedCustomFields: CopperCustomField[] = [ + { + id: 394020, + name: 'Strategic Relationship', + dataType: 'Integration Type', + fieldType: 'option', + }, + { id: 394013, name: 'ERC-20 Exchange', dataType: 'Integration Type', fieldType: 'option' }, + { id: 394014, name: 'ERC-721 Marketplace', dataType: 'Integration Type', fieldType: 'option' }, + { id: 394015, name: 'Trade Widget', dataType: 'Integration Type', fieldType: 'option' }, + { + id: 394016, + name: 'Prediction Market Exchange', + dataType: 'Integration Type', + fieldType: 'option', + }, + { + id: 394017, + name: 'Security Token Exchange', + dataType: 'Integration Type', + fieldType: 'option', + }, + { id: 394018, name: 'Complementary Company', dataType: 'Integration Type', fieldType: 'option' }, + { id: 394019, name: 'Service Provider', dataType: 'Integration Type', fieldType: 'option' }, + { id: 261066, name: 'Integration Type', dataType: 'MultiSelect' }, + { id: 394129, name: 'Market Maker', dataType: 'Company Type', fieldType: 'option' }, + { id: 394130, name: 'Events', dataType: 'Company Type', fieldType: 'option' }, + { id: 394023, name: 'Exchange', dataType: 'Company Type', fieldType: 'option' }, + { id: 394024, name: 'Investor', dataType: 'Company Type', fieldType: 'option' }, + { id: 394026, name: 'Service Provider', dataType: 'Company Type', fieldType: 'option' }, + { id: 394027, name: 'Wallet', dataType: 'Company Type', fieldType: 'option' }, + { id: 394134, name: 'Game', dataType: 'Company Type', fieldType: 'option' }, + { id: 394025, name: 'OTC', dataType: 'Company Type', fieldType: 'option' }, + { id: 394021, name: 'Blockchain/Protocol', dataType: 'Company Type', fieldType: 'option' }, + { id: 394022, name: 'dApp', dataType: 'Company Type', fieldType: 'option' }, + { id: 261067, name: 'Company Type', dataType: 'Dropdown' }, +]; +export { ParsedCustomFields }; diff --git a/packages/pipeline/test/fixtures/copper/api_v1_list_activities.json b/packages/pipeline/test/fixtures/copper/api_v1_list_activities.json new file mode 100644 index 000000000..a726111ac --- /dev/null +++ b/packages/pipeline/test/fixtures/copper/api_v1_list_activities.json @@ -0,0 +1,242 @@ +[ + { + "id": 5015299552, + "parent": { "id": 14667512, "type": "opportunity" }, + "type": { "id": 3, "category": "system", "name": "Stage Change" }, + "user_id": 680302, + "details": "blah blah", + "activity_date": 1545329595, + "old_value": { "id": 2392929, "name": "Evaluation" }, + "new_value": { "id": 2392931, "name": "Integration Started" }, + "date_created": 1545329595, + "date_modified": 1545329595 + }, + { + "id": 5010214065, + "parent": { "id": 14978865, "type": "opportunity" }, + "type": { "id": 3, "category": "system", "name": "Stage Change" }, + "user_id": 680302, + "details": "blah blah", + "activity_date": 1545245706, + "old_value": { "id": 2392928, "name": "Intro" }, + "new_value": { "id": 2392929, "name": "Evaluation" }, + "date_created": 1545245706, + "date_modified": 1545245706 + }, + { + "id": 5006149111, + "parent": { "id": 70430977, "type": "person" }, + "type": { "id": 660495, "category": "user" }, + "user_id": 680302, + "details": "blah blah", + "activity_date": 1545166908, + "old_value": null, + "new_value": null, + "date_created": 1545168280, + "date_modified": 1545166908 + }, + { + "id": 5005314622, + "parent": { "id": 27778968, "type": "company" }, + "type": { "id": 660495, "category": "user" }, + "user_id": 680302, + "details": "blah blah", + "activity_date": 1545080504, + "old_value": null, + "new_value": null, + "date_created": 1545160479, + "date_modified": 1545080504 + }, + { + "id": 5000006802, + "parent": { "id": 14956518, "type": "opportunity" }, + "type": { "id": 660495, "category": "user" }, + "user_id": 680302, + "details": "blah blah", + "activity_date": 1545071374, + "old_value": null, + "new_value": null, + "date_created": 1545071500, + "date_modified": 1545071374 + }, + { + "id": 4985504199, + "parent": { "id": 14912790, "type": "opportunity" }, + "type": { "id": 660495, "category": "user" }, + "user_id": 680302, + "details": "blah blah", + "activity_date": 1544644058, + "old_value": null, + "new_value": null, + "date_created": 1544644661, + "date_modified": 1544644058 + }, + { + "id": 4985456147, + "parent": { "id": 14912790, "type": "opportunity" }, + "type": { "id": 660495, "category": "user" }, + "user_id": 680302, + "details": "blah blah", + "activity_date": 1544644048, + "old_value": null, + "new_value": null, + "date_created": 1544644053, + "date_modified": 1544644048 + }, + { + "id": 4980975996, + "parent": { "id": 14902828, "type": "opportunity" }, + "type": { "id": 660495, "category": "user" }, + "user_id": 680302, + "details": "blah blah", + "activity_date": 1544563171, + "old_value": null, + "new_value": null, + "date_created": 1544563224, + "date_modified": 1544563171 + }, + { + "id": 4980910331, + "parent": { "id": 14902828, "type": "opportunity" }, + "type": { "id": 3, "category": "system", "name": "Stage Change" }, + "user_id": 680302, + "details": "blah blah", + "activity_date": 1544562495, + "old_value": { "id": 2392928, "name": "Intro" }, + "new_value": { "id": 2392931, "name": "Integration Started" }, + "date_created": 1544562495, + "date_modified": 1544562495 + }, + { + "id": 4980872220, + "parent": { "id": 14888910, "type": "opportunity" }, + "type": { "id": 660495, "category": "user" }, + "user_id": 680302, + "details": "blah blah", + "activity_date": 1544559279, + "old_value": null, + "new_value": null, + "date_created": 1544562118, + "date_modified": 1544559279 + }, + { + "id": 4980508097, + "parent": { "id": 14050167, "type": "opportunity" }, + "type": { "id": 1, "category": "system", "name": "Status Change" }, + "user_id": 680302, + "details": "blah blah", + "activity_date": 1544558077, + "old_value": "Open", + "new_value": "Won", + "date_created": 1544558077, + "date_modified": 1544558077 + }, + { + "id": 4980508095, + "parent": { "id": 66538237, "type": "person" }, + "type": { "id": 1, "category": "system" }, + "user_id": 680302, + "details": "blah blah", + "activity_date": 1544558077, + "old_value": null, + "new_value": null, + "date_created": 1544558077, + "date_modified": 1544558077 + }, + { + "id": 4980508092, + "parent": { "id": 27779020, "type": "company" }, + "type": { "id": 1, "category": "system" }, + "user_id": 680302, + "details": "blah blah", + "activity_date": 1544558077, + "old_value": null, + "new_value": null, + "date_created": 1544558077, + "date_modified": 1544558077 + }, + { + "id": 4980507507, + "parent": { "id": 14050167, "type": "opportunity" }, + "type": { "id": 3, "category": "system", "name": "Stage Change" }, + "user_id": 680302, + "details": "blah blah", + "activity_date": 1544558071, + "old_value": { "id": 2392931, "name": "Integration Started" }, + "new_value": { "id": 2405442, "name": "Integration Complete" }, + "date_created": 1544558071, + "date_modified": 1544558071 + }, + { + "id": 4980479684, + "parent": { "id": 14901232, "type": "opportunity" }, + "type": { "id": 3, "category": "system", "name": "Stage Change" }, + "user_id": 680302, + "details": "blah blah", + "activity_date": 1544557777, + "old_value": { "id": 2392928, "name": "Intro" }, + "new_value": { "id": 2392929, "name": "Evaluation" }, + "date_created": 1544557777, + "date_modified": 1544557777 + }, + { + "id": 4980327164, + "parent": { "id": 14901232, "type": "opportunity" }, + "type": { "id": 660495, "category": "user" }, + "user_id": 680302, + "details": "blah blah", + "activity_date": 1544554864, + "old_value": null, + "new_value": null, + "date_created": 1544556132, + "date_modified": 1544554864 + }, + { + "id": 4975270470, + "parent": { "id": 14888744, "type": "opportunity" }, + "type": { "id": 3, "category": "system", "name": "Stage Change" }, + "user_id": 680302, + "details": "blah blah", + "activity_date": 1544469501, + "old_value": { "id": 2392928, "name": "Intro" }, + "new_value": { "id": 2392931, "name": "Integration Started" }, + "date_created": 1544469501, + "date_modified": 1544469501 + }, + { + "id": 4975255523, + "parent": { "id": 64713448, "type": "person" }, + "type": { "id": 1, "category": "system" }, + "user_id": 680302, + "details": "blah blah", + "activity_date": 1544469389, + "old_value": null, + "new_value": null, + "date_created": 1544469389, + "date_modified": 1544469389 + }, + { + "id": 4975255519, + "parent": { "id": 13735617, "type": "opportunity" }, + "type": { "id": 1, "category": "system", "name": "Status Change" }, + "user_id": 680302, + "details": "blah blah", + "activity_date": 1544469388, + "old_value": "Open", + "new_value": "Won", + "date_created": 1544469388, + "date_modified": 1544469388 + }, + { + "id": 4975255514, + "parent": { "id": 27778968, "type": "company" }, + "type": { "id": 1, "category": "system" }, + "user_id": 680302, + "details": "blah blah", + "activity_date": 1544469388, + "old_value": null, + "new_value": null, + "date_created": 1544469388, + "date_modified": 1544469388 + } +] diff --git a/packages/pipeline/test/fixtures/copper/api_v1_list_activities.ts b/packages/pipeline/test/fixtures/copper/api_v1_list_activities.ts new file mode 100644 index 000000000..51ee9ced3 --- /dev/null +++ b/packages/pipeline/test/fixtures/copper/api_v1_list_activities.ts @@ -0,0 +1,305 @@ +import { CopperActivity } from '../../../src/entities'; + +const ParsedActivities: CopperActivity[] = [ + { + id: 5015299552, + parentId: 14667512, + parentType: 'opportunity', + typeId: 3, + typeCategory: 'system', + typeName: 'Stage Change', + userId: 680302, + dateCreated: 1545329595000, + dateModified: 1545329595000, + oldValueId: 2392929, + oldValueName: 'Evaluation', + newValueId: 2392931, + newValueName: 'Integration Started', + }, + { + id: 5010214065, + parentId: 14978865, + parentType: 'opportunity', + typeId: 3, + typeCategory: 'system', + typeName: 'Stage Change', + userId: 680302, + dateCreated: 1545245706000, + dateModified: 1545245706000, + oldValueId: 2392928, + oldValueName: 'Intro', + newValueId: 2392929, + newValueName: 'Evaluation', + }, + { + id: 5006149111, + parentId: 70430977, + parentType: 'person', + typeId: 660495, + typeCategory: 'user', + typeName: undefined, + userId: 680302, + dateCreated: 1545168280000, + dateModified: 1545166908000, + oldValueId: undefined, + oldValueName: undefined, + newValueId: undefined, + newValueName: undefined, + }, + { + id: 5005314622, + parentId: 27778968, + parentType: 'company', + typeId: 660495, + typeCategory: 'user', + typeName: undefined, + userId: 680302, + dateCreated: 1545160479000, + dateModified: 1545080504000, + oldValueId: undefined, + oldValueName: undefined, + newValueId: undefined, + newValueName: undefined, + }, + { + id: 5000006802, + parentId: 14956518, + parentType: 'opportunity', + typeId: 660495, + typeCategory: 'user', + typeName: undefined, + userId: 680302, + dateCreated: 1545071500000, + dateModified: 1545071374000, + oldValueId: undefined, + oldValueName: undefined, + newValueId: undefined, + newValueName: undefined, + }, + { + id: 4985504199, + parentId: 14912790, + parentType: 'opportunity', + typeId: 660495, + typeCategory: 'user', + typeName: undefined, + userId: 680302, + dateCreated: 1544644661000, + dateModified: 1544644058000, + oldValueId: undefined, + oldValueName: undefined, + newValueId: undefined, + newValueName: undefined, + }, + { + id: 4985456147, + parentId: 14912790, + parentType: 'opportunity', + typeId: 660495, + typeCategory: 'user', + typeName: undefined, + userId: 680302, + dateCreated: 1544644053000, + dateModified: 1544644048000, + oldValueId: undefined, + oldValueName: undefined, + newValueId: undefined, + newValueName: undefined, + }, + { + id: 4980975996, + parentId: 14902828, + parentType: 'opportunity', + typeId: 660495, + typeCategory: 'user', + typeName: undefined, + userId: 680302, + dateCreated: 1544563224000, + dateModified: 1544563171000, + oldValueId: undefined, + oldValueName: undefined, + newValueId: undefined, + newValueName: undefined, + }, + { + id: 4980910331, + parentId: 14902828, + parentType: 'opportunity', + typeId: 3, + typeCategory: 'system', + typeName: 'Stage Change', + userId: 680302, + dateCreated: 1544562495000, + dateModified: 1544562495000, + oldValueId: 2392928, + oldValueName: 'Intro', + newValueId: 2392931, + newValueName: 'Integration Started', + }, + { + id: 4980872220, + parentId: 14888910, + parentType: 'opportunity', + typeId: 660495, + typeCategory: 'user', + typeName: undefined, + userId: 680302, + dateCreated: 1544562118000, + dateModified: 1544559279000, + oldValueId: undefined, + oldValueName: undefined, + newValueId: undefined, + newValueName: undefined, + }, + { + id: 4980508097, + parentId: 14050167, + parentType: 'opportunity', + typeId: 1, + typeCategory: 'system', + typeName: 'Status Change', + userId: 680302, + dateCreated: 1544558077000, + dateModified: 1544558077000, + oldValueId: undefined, + oldValueName: undefined, + newValueId: undefined, + newValueName: undefined, + }, + { + id: 4980508095, + parentId: 66538237, + parentType: 'person', + typeId: 1, + typeCategory: 'system', + typeName: undefined, + userId: 680302, + dateCreated: 1544558077000, + dateModified: 1544558077000, + oldValueId: undefined, + oldValueName: undefined, + newValueId: undefined, + newValueName: undefined, + }, + { + id: 4980508092, + parentId: 27779020, + parentType: 'company', + typeId: 1, + typeCategory: 'system', + typeName: undefined, + userId: 680302, + dateCreated: 1544558077000, + dateModified: 1544558077000, + oldValueId: undefined, + oldValueName: undefined, + newValueId: undefined, + newValueName: undefined, + }, + { + id: 4980507507, + parentId: 14050167, + parentType: 'opportunity', + typeId: 3, + typeCategory: 'system', + typeName: 'Stage Change', + userId: 680302, + dateCreated: 1544558071000, + dateModified: 1544558071000, + oldValueId: 2392931, + oldValueName: 'Integration Started', + newValueId: 2405442, + newValueName: 'Integration Complete', + }, + { + id: 4980479684, + parentId: 14901232, + parentType: 'opportunity', + typeId: 3, + typeCategory: 'system', + typeName: 'Stage Change', + userId: 680302, + dateCreated: 1544557777000, + dateModified: 1544557777000, + oldValueId: 2392928, + oldValueName: 'Intro', + newValueId: 2392929, + newValueName: 'Evaluation', + }, + { + id: 4980327164, + parentId: 14901232, + parentType: 'opportunity', + typeId: 660495, + typeCategory: 'user', + typeName: undefined, + userId: 680302, + dateCreated: 1544556132000, + dateModified: 1544554864000, + oldValueId: undefined, + oldValueName: undefined, + newValueId: undefined, + newValueName: undefined, + }, + { + id: 4975270470, + parentId: 14888744, + parentType: 'opportunity', + typeId: 3, + typeCategory: 'system', + typeName: 'Stage Change', + userId: 680302, + dateCreated: 1544469501000, + dateModified: 1544469501000, + oldValueId: 2392928, + oldValueName: 'Intro', + newValueId: 2392931, + newValueName: 'Integration Started', + }, + { + id: 4975255523, + parentId: 64713448, + parentType: 'person', + typeId: 1, + typeCategory: 'system', + typeName: undefined, + userId: 680302, + dateCreated: 1544469389000, + dateModified: 1544469389000, + oldValueId: undefined, + oldValueName: undefined, + newValueId: undefined, + newValueName: undefined, + }, + { + id: 4975255519, + parentId: 13735617, + parentType: 'opportunity', + typeId: 1, + typeCategory: 'system', + typeName: 'Status Change', + userId: 680302, + dateCreated: 1544469388000, + dateModified: 1544469388000, + oldValueId: undefined, + oldValueName: undefined, + newValueId: undefined, + newValueName: undefined, + }, + { + id: 4975255514, + parentId: 27778968, + parentType: 'company', + typeId: 1, + typeCategory: 'system', + typeName: undefined, + userId: 680302, + dateCreated: 1544469388000, + dateModified: 1544469388000, + oldValueId: undefined, + oldValueName: undefined, + newValueId: undefined, + newValueName: undefined, + }, +]; +export { ParsedActivities }; diff --git a/packages/pipeline/test/fixtures/copper/api_v1_list_leads.json b/packages/pipeline/test/fixtures/copper/api_v1_list_leads.json new file mode 100644 index 000000000..5223976f9 --- /dev/null +++ b/packages/pipeline/test/fixtures/copper/api_v1_list_leads.json @@ -0,0 +1,583 @@ +[ + { + "id": 9150547, + "name": "My Contact", + "prefix": null, + "first_name": "My", + "last_name": "Contact", + "middle_name": null, + "suffix": null, + "address": null, + "assignee_id": null, + "company_name": null, + "customer_source_id": null, + "details": null, + "email": { + "email": "mycontact@noemail.com", + "category": "work" + }, + "interaction_count": 0, + "monetary_value": null, + "socials": [], + "status": "New", + "status_id": 208231, + "tags": [], + "title": null, + "websites": [], + "phone_numbers": [], + "custom_fields": [ + { + "custom_field_definition_id": 100764, + "value": null + }, + { + "custom_field_definition_id": 103481, + "value": null + } + ], + "date_created": 1490045162, + "date_modified": 1490045162 + }, + { + "id": 9150552, + "name": "My Contact", + "prefix": null, + "first_name": "My", + "last_name": "Contact", + "middle_name": null, + "suffix": null, + "address": null, + "assignee_id": null, + "company_name": null, + "customer_source_id": null, + "details": null, + "email": null, + "interaction_count": 0, + "monetary_value": null, + "socials": [], + "status": "New", + "status_id": 208231, + "tags": [], + "title": null, + "websites": [], + "phone_numbers": [ + { + "number": "415-123-45678", + "category": "mobile" + } + ], + "custom_fields": [ + { + "custom_field_definition_id": 100764, + "value": null + }, + { + "custom_field_definition_id": 103481, + "value": null + } + ], + "date_created": 1490045237, + "date_modified": 1490045237 + }, + { + "id": 9150578, + "name": "My Contact", + "prefix": null, + "first_name": "My", + "last_name": "Contact", + "middle_name": null, + "suffix": null, + "address": null, + "assignee_id": null, + "company_name": null, + "customer_source_id": null, + "details": null, + "email": null, + "interaction_count": 0, + "monetary_value": null, + "socials": [], + "status": "New", + "status_id": 208231, + "tags": [], + "title": null, + "websites": [], + "phone_numbers": [ + { + "number": "415-123-45678", + "category": "mobile" + } + ], + "custom_fields": [ + { + "custom_field_definition_id": 100764, + "value": null + }, + { + "custom_field_definition_id": 103481, + "value": null + } + ], + "date_created": 1490045279, + "date_modified": 1490045279 + }, + { + "id": 8982554, + "name": "My Lead", + "prefix": null, + "first_name": "My", + "last_name": "Lead", + "middle_name": null, + "suffix": null, + "address": null, + "assignee_id": null, + "company_name": null, + "customer_source_id": null, + "details": null, + "email": { + "email": "mylead@noemail.com", + "category": "work" + }, + "interaction_count": 0, + "monetary_value": null, + "socials": [], + "status": "New", + "status_id": 208231, + "tags": [], + "title": null, + "websites": [], + "phone_numbers": [], + "custom_fields": [ + { + "custom_field_definition_id": 100764, + "value": null + }, + { + "custom_field_definition_id": 103481, + "value": null + } + ], + "date_created": 1489528899, + "date_modified": 1489528899 + }, + { + "id": 8982702, + "name": "My Lead", + "prefix": null, + "first_name": "My", + "last_name": "Lead", + "middle_name": null, + "suffix": null, + "address": null, + "assignee_id": null, + "company_name": null, + "customer_source_id": null, + "details": null, + "email": { + "email": "mylead@gmail.test", + "category": "work" + }, + "interaction_count": 0, + "monetary_value": null, + "socials": [], + "status": "New", + "status_id": 208231, + "tags": [], + "title": null, + "websites": [], + "phone_numbers": [], + "custom_fields": [ + { + "custom_field_definition_id": 100764, + "value": null + }, + { + "custom_field_definition_id": 103481, + "value": null + } + ], + "date_created": 1489531171, + "date_modified": 1489531171 + }, + { + "id": 9094361, + "name": "My Lead", + "prefix": null, + "first_name": "My", + "last_name": "Lead", + "middle_name": null, + "suffix": null, + "address": null, + "assignee_id": null, + "company_name": null, + "customer_source_id": null, + "details": null, + "email": { + "email": "mylead@noemail.com", + "category": "work" + }, + "interaction_count": 0, + "monetary_value": null, + "socials": [], + "status": "New", + "status_id": 208231, + "tags": [], + "title": null, + "websites": [], + "phone_numbers": [], + "custom_fields": [ + { + "custom_field_definition_id": 100764, + "value": null + }, + { + "custom_field_definition_id": 103481, + "value": null + } + ], + "date_created": 1489791225, + "date_modified": 1489791225 + }, + { + "id": 9094364, + "name": "My Lead", + "prefix": null, + "first_name": "My", + "last_name": "Lead", + "middle_name": null, + "suffix": null, + "address": null, + "assignee_id": null, + "company_name": null, + "customer_source_id": null, + "details": null, + "email": { + "email": "mylead@noemail.com", + "category": "work" + }, + "interaction_count": 0, + "monetary_value": null, + "socials": [], + "status": "New", + "status_id": 208231, + "tags": [], + "title": null, + "websites": [], + "phone_numbers": [], + "custom_fields": [ + { + "custom_field_definition_id": 100764, + "value": "123456789012345678901234567890" + }, + { + "custom_field_definition_id": 103481, + "value": "123456789012345678901234567890" + } + ], + "date_created": 1489791283, + "date_modified": 1489791283 + }, + { + "id": 9094371, + "name": "My Lead", + "prefix": null, + "first_name": "My", + "last_name": "Lead", + "middle_name": null, + "suffix": null, + "address": null, + "assignee_id": null, + "company_name": null, + "customer_source_id": null, + "details": null, + "email": { + "email": "mylead@noemail.com", + "category": "work" + }, + "interaction_count": 0, + "monetary_value": null, + "socials": [], + "status": "New", + "status_id": 208231, + "tags": [], + "title": null, + "websites": [], + "phone_numbers": [], + "custom_fields": [ + { + "custom_field_definition_id": 100764, + "value": + "|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------" + }, + { + "custom_field_definition_id": 103481, + "value": "123456789012345678901234567890" + } + ], + "date_created": 1489791417, + "date_modified": 1489791417 + }, + { + "id": 9094372, + "name": "My Lead", + "prefix": null, + "first_name": "My", + "last_name": "Lead", + "middle_name": null, + "suffix": null, + "address": null, + "assignee_id": null, + "company_name": null, + "customer_source_id": null, + "details": null, + "email": { + "email": "mylead@noemail.com", + "category": "work" + }, + "interaction_count": 0, + "monetary_value": null, + "socials": [], + "status": "New", + "status_id": 208231, + "tags": [], + "title": null, + "websites": [], + "phone_numbers": [], + "custom_fields": [ + { + "custom_field_definition_id": 100764, + "value": + "|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------|--------1---------2---------3---------4---------5-----" + }, + { + "custom_field_definition_id": 103481, + "value": "123456789012345678901234567890" + } + ], + "date_created": 1489791453, + "date_modified": 1489791453 + }, + { + "id": 9094373, + "name": "My Lead", + "prefix": null, + "first_name": "My", + "last_name": "Lead", + "middle_name": null, + "suffix": null, + "address": null, + "assignee_id": null, + "company_name": null, + "customer_source_id": null, + "details": null, + "email": { + "email": "mylead@noemail.com", + "category": "work" + }, + "interaction_count": 0, + "monetary_value": null, + "socials": [], + "status": "New", + "status_id": 208231, + "tags": [], + "title": null, + "websites": [], + "phone_numbers": [], + "custom_fields": [ + { + "custom_field_definition_id": 100764, + "value": + "|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------|--------1---------2---------3---------4---------5-----" + }, + { + "custom_field_definition_id": 103481, + "value": + "|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------" + } + ], + "date_created": 1489791470, + "date_modified": 1489791470 + }, + { + "id": 9094383, + "name": "My Lead", + "prefix": null, + "first_name": "My", + "last_name": "Lead", + "middle_name": null, + "suffix": null, + "address": null, + "assignee_id": null, + "company_name": null, + "customer_source_id": null, + "details": null, + "email": { + "email": "mylead@noemail.com", + "category": "work" + }, + "interaction_count": 0, + "monetary_value": null, + "socials": [], + "status": "New", + "status_id": 208231, + "tags": [], + "title": null, + "websites": [], + "phone_numbers": [], + "custom_fields": [ + { + "custom_field_definition_id": 100764, + "value": + "|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------|--------1---------2---------3---------4---------5-----" + }, + { + "custom_field_definition_id": 103481, + "value": + "|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------" + } + ], + "date_created": 1489791672, + "date_modified": 1489791672 + }, + { + "id": 9174441, + "name": "My Lead", + "prefix": null, + "first_name": "My", + "last_name": "Lead", + "middle_name": null, + "suffix": null, + "address": null, + "assignee_id": null, + "company_name": null, + "customer_source_id": null, + "details": null, + "email": { + "email": "mylead@noemail.com", + "category": "work" + }, + "interaction_count": 0, + "monetary_value": null, + "socials": [], + "status": "New", + "status_id": 208231, + "tags": [], + "title": null, + "websites": [], + "phone_numbers": [], + "custom_fields": [ + { + "custom_field_definition_id": 100764, + "value": "Text fields are 255 chars or less!" + }, + { + "custom_field_definition_id": 103481, + "value": "text \n text" + } + ], + "date_created": 1490112942, + "date_modified": 1490112942 + }, + { + "id": 9174443, + "name": "My Lead", + "prefix": null, + "first_name": "My", + "last_name": "Lead", + "middle_name": null, + "suffix": null, + "address": null, + "assignee_id": null, + "company_name": null, + "customer_source_id": null, + "details": null, + "email": { + "email": "mylead@noemail.com", + "category": "work" + }, + "interaction_count": 0, + "monetary_value": null, + "socials": [], + "status": "New", + "status_id": 208231, + "tags": [], + "title": null, + "websites": [], + "phone_numbers": [], + "custom_fields": [ + { + "custom_field_definition_id": 100764, + "value": "Text fields are 255 chars or less!" + }, + { + "custom_field_definition_id": 103481, + "value": "text /n text" + } + ], + "date_created": 1490112953, + "date_modified": 1490112953 + }, + { + "id": 8894157, + "name": "Test Lead", + "prefix": null, + "first_name": "Test", + "last_name": "Lead", + "middle_name": null, + "suffix": null, + "address": { + "street": "301 Howard St Ste 600", + "city": "San Francisco", + "state": "CA", + "postal_code": "94105", + "country": "US" + }, + "assignee_id": 137658, + "company_name": "Lead's Company", + "customer_source_id": 331241, + "details": "This is an update", + "email": { + "email": "address@workemail.com", + "category": "work" + }, + "interaction_count": 0, + "monetary_value": 100, + "socials": [ + { + "url": "facebook.com/test_lead", + "category": "facebook" + } + ], + "status": "New", + "status_id": 208231, + "tags": ["tag 1", "tag 2"], + "title": "Title", + "websites": [ + { + "url": "www.workwebsite.com", + "category": "work" + } + ], + "phone_numbers": [ + { + "number": "415-999-4321", + "category": "mobile" + }, + { + "number": "415-555-1234", + "category": "work" + } + ], + "custom_fields": [ + { + "custom_field_definition_id": 100764, + "value": null + }, + { + "custom_field_definition_id": 103481, + "value": null + } + ], + "date_created": 1489018784, + "date_modified": 1496692911 + } +] diff --git a/packages/pipeline/test/fixtures/copper/api_v1_list_leads.ts b/packages/pipeline/test/fixtures/copper/api_v1_list_leads.ts new file mode 100644 index 000000000..b1f00cba7 --- /dev/null +++ b/packages/pipeline/test/fixtures/copper/api_v1_list_leads.ts @@ -0,0 +1,229 @@ +import { CopperLead } from '../../../src/entities'; +const ParsedLeads: CopperLead[] = [ + { + id: 9150547, + name: 'My Contact', + firstName: 'My', + lastName: 'Contact', + middleName: undefined, + assigneeId: undefined, + companyName: undefined, + customerSourceId: undefined, + monetaryValue: undefined, + status: 'New', + statusId: 208231, + title: undefined, + dateCreated: 1490045162000, + dateModified: 1490045162000, + }, + { + id: 9150552, + name: 'My Contact', + firstName: 'My', + lastName: 'Contact', + middleName: undefined, + assigneeId: undefined, + companyName: undefined, + customerSourceId: undefined, + monetaryValue: undefined, + status: 'New', + statusId: 208231, + title: undefined, + dateCreated: 1490045237000, + dateModified: 1490045237000, + }, + { + id: 9150578, + name: 'My Contact', + firstName: 'My', + lastName: 'Contact', + middleName: undefined, + assigneeId: undefined, + companyName: undefined, + customerSourceId: undefined, + monetaryValue: undefined, + status: 'New', + statusId: 208231, + title: undefined, + dateCreated: 1490045279000, + dateModified: 1490045279000, + }, + { + id: 8982554, + name: 'My Lead', + firstName: 'My', + lastName: 'Lead', + middleName: undefined, + assigneeId: undefined, + companyName: undefined, + customerSourceId: undefined, + monetaryValue: undefined, + status: 'New', + statusId: 208231, + title: undefined, + dateCreated: 1489528899000, + dateModified: 1489528899000, + }, + { + id: 8982702, + name: 'My Lead', + firstName: 'My', + lastName: 'Lead', + middleName: undefined, + assigneeId: undefined, + companyName: undefined, + customerSourceId: undefined, + monetaryValue: undefined, + status: 'New', + statusId: 208231, + title: undefined, + dateCreated: 1489531171000, + dateModified: 1489531171000, + }, + { + id: 9094361, + name: 'My Lead', + firstName: 'My', + lastName: 'Lead', + middleName: undefined, + assigneeId: undefined, + companyName: undefined, + customerSourceId: undefined, + monetaryValue: undefined, + status: 'New', + statusId: 208231, + title: undefined, + dateCreated: 1489791225000, + dateModified: 1489791225000, + }, + { + id: 9094364, + name: 'My Lead', + firstName: 'My', + lastName: 'Lead', + middleName: undefined, + assigneeId: undefined, + companyName: undefined, + customerSourceId: undefined, + monetaryValue: undefined, + status: 'New', + statusId: 208231, + title: undefined, + dateCreated: 1489791283000, + dateModified: 1489791283000, + }, + { + id: 9094371, + name: 'My Lead', + firstName: 'My', + lastName: 'Lead', + middleName: undefined, + assigneeId: undefined, + companyName: undefined, + customerSourceId: undefined, + monetaryValue: undefined, + status: 'New', + statusId: 208231, + title: undefined, + dateCreated: 1489791417000, + dateModified: 1489791417000, + }, + { + id: 9094372, + name: 'My Lead', + firstName: 'My', + lastName: 'Lead', + middleName: undefined, + assigneeId: undefined, + companyName: undefined, + customerSourceId: undefined, + monetaryValue: undefined, + status: 'New', + statusId: 208231, + title: undefined, + dateCreated: 1489791453000, + dateModified: 1489791453000, + }, + { + id: 9094373, + name: 'My Lead', + firstName: 'My', + lastName: 'Lead', + middleName: undefined, + assigneeId: undefined, + companyName: undefined, + customerSourceId: undefined, + monetaryValue: undefined, + status: 'New', + statusId: 208231, + title: undefined, + dateCreated: 1489791470000, + dateModified: 1489791470000, + }, + { + id: 9094383, + name: 'My Lead', + firstName: 'My', + lastName: 'Lead', + middleName: undefined, + assigneeId: undefined, + companyName: undefined, + customerSourceId: undefined, + monetaryValue: undefined, + status: 'New', + statusId: 208231, + title: undefined, + dateCreated: 1489791672000, + dateModified: 1489791672000, + }, + { + id: 9174441, + name: 'My Lead', + firstName: 'My', + lastName: 'Lead', + middleName: undefined, + assigneeId: undefined, + companyName: undefined, + customerSourceId: undefined, + monetaryValue: undefined, + status: 'New', + statusId: 208231, + title: undefined, + dateCreated: 1490112942000, + dateModified: 1490112942000, + }, + { + id: 9174443, + name: 'My Lead', + firstName: 'My', + lastName: 'Lead', + middleName: undefined, + assigneeId: undefined, + companyName: undefined, + customerSourceId: undefined, + monetaryValue: undefined, + status: 'New', + statusId: 208231, + title: undefined, + dateCreated: 1490112953000, + dateModified: 1490112953000, + }, + { + id: 8894157, + name: 'Test Lead', + firstName: 'Test', + lastName: 'Lead', + middleName: undefined, + assigneeId: 137658, + companyName: "Lead's Company", + customerSourceId: 331241, + monetaryValue: 100, + status: 'New', + statusId: 208231, + title: 'Title', + dateCreated: 1489018784000, + dateModified: 1496692911000, + }, +]; + +export { ParsedLeads }; diff --git a/packages/pipeline/test/fixtures/copper/api_v1_list_opportunities.json b/packages/pipeline/test/fixtures/copper/api_v1_list_opportunities.json new file mode 100644 index 000000000..34ac58c30 --- /dev/null +++ b/packages/pipeline/test/fixtures/copper/api_v1_list_opportunities.json @@ -0,0 +1,662 @@ +[ + { + "id": 14050269, + "name": "8Base RaaS", + "assignee_id": 680302, + "close_date": "11/19/2018", + "company_id": 27778962, + "company_name": "8base", + "customer_source_id": null, + "details": "blah blah", + "loss_reason_id": null, + "pipeline_id": 512676, + "pipeline_stage_id": 2405442, + "primary_contact_id": 66088850, + "priority": "None", + "status": "Won", + "tags": [], + "interaction_count": 81, + "monetary_unit": null, + "monetary_value": null, + "converted_unit": null, + "converted_value": null, + "win_probability": 0, + "date_stage_changed": 1542653860, + "date_last_contacted": 1544757550, + "leads_converted_from": [], + "date_lead_created": null, + "date_created": 1538414159, + "date_modified": 1544769562, + "custom_fields": [ + { "custom_field_definition_id": 261066, "value": [394013, 394018] }, + { "custom_field_definition_id": 261067, "value": 394026 } + ] + }, + { + "id": 14631430, + "name": "Alice.si TW + ERC 20 Marketplace", + "assignee_id": 680302, + "close_date": "12/15/2018", + "company_id": 30238847, + "company_name": "Alice SI", + "customer_source_id": null, + "details": "blah blah", + "loss_reason_id": null, + "pipeline_id": 512676, + "pipeline_stage_id": 2392929, + "primary_contact_id": 69354024, + "priority": "None", + "status": "Open", + "tags": [], + "interaction_count": 4, + "monetary_unit": null, + "monetary_value": null, + "converted_unit": null, + "converted_value": null, + "win_probability": 0, + "date_stage_changed": 1542304481, + "date_last_contacted": 1542304800, + "leads_converted_from": [], + "date_lead_created": null, + "date_created": 1542304481, + "date_modified": 1542304943, + "custom_fields": [ + { "custom_field_definition_id": 261066, "value": [394013, 394015] }, + { "custom_field_definition_id": 261067, "value": 394023 } + ] + }, + { + "id": 14632057, + "name": "Altcoin.io Relayer", + "assignee_id": 680302, + "close_date": "12/15/2018", + "company_id": 29936486, + "company_name": "Altcoin.io", + "customer_source_id": null, + "details": "blah blah", + "loss_reason_id": null, + "pipeline_id": 512676, + "pipeline_stage_id": 2392929, + "primary_contact_id": 68724646, + "priority": "None", + "status": "Open", + "tags": [], + "interaction_count": 22, + "monetary_unit": null, + "monetary_value": null, + "converted_unit": null, + "converted_value": null, + "win_probability": 0, + "date_stage_changed": 1542310909, + "date_last_contacted": 1543864597, + "leads_converted_from": [], + "date_lead_created": null, + "date_created": 1542306827, + "date_modified": 1543864667, + "custom_fields": [ + { "custom_field_definition_id": 261066, "value": [394013, 394017] }, + { "custom_field_definition_id": 261067, "value": 394023 } + ] + }, + { + "id": 14667523, + "name": "Altcoin.io Relayer", + "assignee_id": 680302, + "close_date": "12/19/2018", + "company_id": 29936486, + "company_name": "Altcoin.io", + "customer_source_id": null, + "details": "blah blah", + "loss_reason_id": null, + "pipeline_id": 512676, + "pipeline_stage_id": 2392929, + "primary_contact_id": 68724646, + "priority": "None", + "status": "Open", + "tags": [], + "interaction_count": 21, + "monetary_unit": null, + "monetary_value": null, + "converted_unit": null, + "converted_value": null, + "win_probability": 0, + "date_stage_changed": 1542657437, + "date_last_contacted": 1543864597, + "leads_converted_from": [], + "date_lead_created": null, + "date_created": 1542657437, + "date_modified": 1543864667, + "custom_fields": [ + { "custom_field_definition_id": 261066, "value": [394013, 394017] }, + { "custom_field_definition_id": 261067, "value": 394023 } + ] + }, + { + "id": 14666706, + "name": "Amadeus Relayer", + "assignee_id": 680302, + "close_date": "11/19/2018", + "company_id": 29243209, + "company_name": "Amadeus", + "customer_source_id": null, + "details": "blah blah", + "loss_reason_id": null, + "pipeline_id": 512676, + "pipeline_stage_id": 2405442, + "primary_contact_id": 66912020, + "priority": "None", + "status": "Won", + "tags": [], + "interaction_count": 11, + "monetary_unit": null, + "monetary_value": null, + "converted_unit": null, + "converted_value": null, + "win_probability": 0, + "date_stage_changed": 1542654284, + "date_last_contacted": 1543264254, + "leads_converted_from": [], + "date_lead_created": null, + "date_created": 1542654284, + "date_modified": 1543277520, + "custom_fields": [ + { "custom_field_definition_id": 261066, "value": [394013] }, + { "custom_field_definition_id": 261067, "value": 394023 } + ] + }, + { + "id": 14666718, + "name": "Ambo Relayer", + "assignee_id": 680302, + "close_date": "11/19/2018", + "company_id": 29249190, + "company_name": "Ambo", + "customer_source_id": null, + "details": "blah blah", + "loss_reason_id": null, + "pipeline_id": 512676, + "pipeline_stage_id": 2405442, + "primary_contact_id": 66927869, + "priority": "None", + "status": "Won", + "tags": [], + "interaction_count": 126, + "monetary_unit": null, + "monetary_value": null, + "converted_unit": null, + "converted_value": null, + "win_probability": 0, + "date_stage_changed": 1542654352, + "date_last_contacted": 1545252349, + "leads_converted_from": [], + "date_lead_created": null, + "date_created": 1542654352, + "date_modified": 1545253761, + "custom_fields": [ + { "custom_field_definition_id": 261066, "value": [394013] }, + { "custom_field_definition_id": 261067, "value": 394023 } + ] + }, + { + "id": 14164318, + "name": "Augur TW", + "assignee_id": 680302, + "close_date": "12/10/2018", + "company_id": 27778967, + "company_name": "Augur", + "customer_source_id": null, + "details": "blah blah", + "loss_reason_id": null, + "pipeline_id": 512676, + "pipeline_stage_id": 2405442, + "primary_contact_id": 67248692, + "priority": "None", + "status": "Won", + "tags": [], + "interaction_count": 22, + "monetary_unit": null, + "monetary_value": null, + "converted_unit": null, + "converted_value": null, + "win_probability": 0, + "date_stage_changed": 1544469362, + "date_last_contacted": 1544491567, + "leads_converted_from": [], + "date_lead_created": null, + "date_created": 1539204858, + "date_modified": 1544653867, + "custom_fields": [ + { "custom_field_definition_id": 261066, "value": [394015] }, + { "custom_field_definition_id": 261067, "value": 394021 } + ] + }, + { + "id": 14666626, + "name": "Autonio", + "assignee_id": 680302, + "close_date": "12/19/2018", + "company_id": 27920701, + "company_name": "Auton", + "customer_source_id": null, + "details": "blah blah", + "loss_reason_id": null, + "pipeline_id": 512676, + "pipeline_stage_id": 2392931, + "primary_contact_id": 64742640, + "priority": "None", + "status": "Open", + "tags": [], + "interaction_count": 54, + "monetary_unit": null, + "monetary_value": null, + "converted_unit": null, + "converted_value": null, + "win_probability": 0, + "date_stage_changed": 1542653834, + "date_last_contacted": 1542658568, + "leads_converted_from": [], + "date_lead_created": null, + "date_created": 1542653834, + "date_modified": 1542658808, + "custom_fields": [ + { "custom_field_definition_id": 261066, "value": [394013, 394019] }, + { "custom_field_definition_id": 261067, "value": 394023 } + ] + }, + { + "id": 14050921, + "name": "Axie Infinity 721 Marketplace", + "assignee_id": 680302, + "close_date": "11/1/2018", + "company_id": 27779033, + "company_name": "Axie Infinity", + "customer_source_id": null, + "details": "blah blah", + "loss_reason_id": null, + "pipeline_id": 512676, + "pipeline_stage_id": 2392931, + "primary_contact_id": 66499254, + "priority": "None", + "status": "Open", + "tags": [], + "interaction_count": 4, + "monetary_unit": null, + "monetary_value": null, + "converted_unit": null, + "converted_value": null, + "win_probability": 0, + "date_stage_changed": 1543861025, + "date_last_contacted": 1539024738, + "leads_converted_from": [], + "date_lead_created": null, + "date_created": 1538416687, + "date_modified": 1543861025, + "custom_fields": [ + { "custom_field_definition_id": 261066, "value": [394014] }, + { "custom_field_definition_id": 261067, "value": 394134 } + ] + }, + { + "id": 13735617, + "name": "Balance TW", + "assignee_id": 680302, + "close_date": "12/10/2018", + "company_id": 27778968, + "company_name": "Balance", + "customer_source_id": null, + "details": "blah blah", + "loss_reason_id": null, + "pipeline_id": 512676, + "pipeline_stage_id": 2405442, + "primary_contact_id": 64713448, + "priority": "None", + "status": "Won", + "tags": [], + "interaction_count": 34, + "monetary_unit": null, + "monetary_value": null, + "converted_unit": null, + "converted_value": null, + "win_probability": 0, + "date_stage_changed": 1544469382, + "date_last_contacted": 1545082200, + "leads_converted_from": [], + "date_lead_created": null, + "date_created": 1535668009, + "date_modified": 1545082454, + "custom_fields": [ + { "custom_field_definition_id": 261066, "value": [394015] }, + { "custom_field_definition_id": 261067, "value": 394027 } + ] + }, + { + "id": 14667112, + "name": "Bamboo Relayer", + "assignee_id": 680302, + "close_date": "11/19/2018", + "company_id": 29243795, + "company_name": "Bamboo Relay", + "customer_source_id": null, + "details": "blah blah", + "loss_reason_id": null, + "pipeline_id": 512676, + "pipeline_stage_id": 2405442, + "primary_contact_id": 66914687, + "priority": "None", + "status": "Won", + "tags": [], + "interaction_count": 46, + "monetary_unit": null, + "monetary_value": null, + "converted_unit": null, + "converted_value": null, + "win_probability": 0, + "date_stage_changed": 1542655143, + "date_last_contacted": 1545252349, + "leads_converted_from": [], + "date_lead_created": null, + "date_created": 1542655143, + "date_modified": 1545253761, + "custom_fields": [ + { "custom_field_definition_id": 261066, "value": [394013] }, + { "custom_field_definition_id": 261067, "value": 394023 } + ] + }, + { + "id": 13627309, + "name": "Ben TW", + "assignee_id": 680302, + "close_date": "1/1/2019", + "company_id": 27702348, + "company_name": "Ben", + "customer_source_id": null, + "details": "blah blah", + "loss_reason_id": null, + "pipeline_id": 512676, + "pipeline_stage_id": 2392929, + "primary_contact_id": 64262622, + "priority": "None", + "status": "Open", + "tags": [], + "interaction_count": 64, + "monetary_unit": null, + "monetary_value": null, + "converted_unit": null, + "converted_value": null, + "win_probability": 0, + "date_stage_changed": 1541527279, + "date_last_contacted": 1541639882, + "leads_converted_from": [], + "date_lead_created": null, + "date_created": 1534887789, + "date_modified": 1541651395, + "custom_fields": [ + { "custom_field_definition_id": 261066, "value": [394015] }, + { "custom_field_definition_id": 261067, "value": 394027 } + ] + }, + { + "id": 14808512, + "name": "Bit2Me Relayer", + "assignee_id": 680302, + "close_date": "12/3/2018", + "company_id": 30793050, + "company_name": "Bit2Me", + "customer_source_id": null, + "details": "blah blah", + "loss_reason_id": null, + "pipeline_id": 512676, + "pipeline_stage_id": 2405442, + "primary_contact_id": 70267217, + "priority": "None", + "status": "Won", + "tags": [], + "interaction_count": 0, + "monetary_unit": null, + "monetary_value": null, + "converted_unit": null, + "converted_value": null, + "win_probability": 0, + "date_stage_changed": 1543861167, + "date_last_contacted": null, + "leads_converted_from": [], + "date_lead_created": null, + "date_created": 1543861167, + "date_modified": 1543861189, + "custom_fields": [ + { "custom_field_definition_id": 261066, "value": [394013] }, + { "custom_field_definition_id": 261067, "value": 394023 } + ] + }, + { + "id": 14050312, + "name": "Bitcoin.tax Reporting Integration", + "assignee_id": 680302, + "close_date": "11/1/2018", + "company_id": 27957614, + "company_name": "Bitcoin", + "customer_source_id": null, + "details": "blah blah", + "loss_reason_id": null, + "pipeline_id": 512676, + "pipeline_stage_id": 2392928, + "primary_contact_id": 66539479, + "priority": "None", + "status": "Open", + "tags": [], + "interaction_count": 5, + "monetary_unit": null, + "monetary_value": null, + "converted_unit": null, + "converted_value": null, + "win_probability": 0, + "date_stage_changed": 1538414308, + "date_last_contacted": 1536766098, + "leads_converted_from": [], + "date_lead_created": null, + "date_created": 1538414308, + "date_modified": 1538414314, + "custom_fields": [ + { "custom_field_definition_id": 261066, "value": [394019] }, + { "custom_field_definition_id": 261067, "value": 394026 } + ] + }, + { + "id": 14331463, + "name": "Bitpie TW", + "assignee_id": 680302, + "close_date": "11/19/2018", + "company_id": 27779026, + "company_name": "Bitpie", + "customer_source_id": null, + "details": "blah blah", + "loss_reason_id": null, + "pipeline_id": 512676, + "pipeline_stage_id": 2392929, + "primary_contact_id": 67700943, + "priority": "None", + "status": "Open", + "tags": [], + "interaction_count": 9, + "monetary_unit": null, + "monetary_value": null, + "converted_unit": null, + "converted_value": null, + "win_probability": 0, + "date_stage_changed": 1539984566, + "date_last_contacted": 1541529947, + "leads_converted_from": [], + "date_lead_created": null, + "date_created": 1539984566, + "date_modified": 1541530233, + "custom_fields": [ + { "custom_field_definition_id": 261066, "value": [394015] }, + { "custom_field_definition_id": 261067, "value": 394027 } + ] + }, + { + "id": 14331481, + "name": "Bitski Wallet SDK TW", + "assignee_id": 680302, + "close_date": "11/19/2018", + "company_id": 29489300, + "company_name": "Bitski", + "customer_source_id": null, + "details": "blah blah", + "loss_reason_id": null, + "pipeline_id": 512676, + "pipeline_stage_id": 2392929, + "primary_contact_id": 67697528, + "priority": "None", + "status": "Open", + "tags": [], + "interaction_count": 23, + "monetary_unit": null, + "monetary_value": null, + "converted_unit": null, + "converted_value": null, + "win_probability": 0, + "date_stage_changed": 1539984735, + "date_last_contacted": 1544811399, + "leads_converted_from": [], + "date_lead_created": null, + "date_created": 1539984735, + "date_modified": 1544818605, + "custom_fields": [ + { "custom_field_definition_id": 261066, "value": [394015] }, + { "custom_field_definition_id": 261067, "value": 394026 } + ] + }, + { + "id": 14531554, + "name": "BitUniverse TW", + "assignee_id": 680302, + "close_date": "12/6/2018", + "company_id": 29901805, + "company_name": "BitUniverse Co., Ltd (Cryptocurrency Portfolio)", + "customer_source_id": null, + "details": "blah blah", + "loss_reason_id": null, + "pipeline_id": 512676, + "pipeline_stage_id": 2392929, + "primary_contact_id": 68692107, + "priority": "None", + "status": "Open", + "tags": [], + "interaction_count": 15, + "monetary_unit": null, + "monetary_value": null, + "converted_unit": null, + "converted_value": null, + "win_probability": 0, + "date_stage_changed": 1543861104, + "date_last_contacted": 1544803276, + "leads_converted_from": [], + "date_lead_created": null, + "date_created": 1541527110, + "date_modified": 1544812979, + "custom_fields": [ + { "custom_field_definition_id": 261066, "value": [394015] }, + { "custom_field_definition_id": 261067, "value": 394026 } + ] + }, + { + "id": 14050895, + "name": "BlitzPredict PMR", + "assignee_id": 680302, + "close_date": "11/1/2018", + "company_id": 28758258, + "company_name": "BlitzPredict", + "customer_source_id": null, + "details": "blah blah", + "loss_reason_id": null, + "pipeline_id": 512676, + "pipeline_stage_id": 2392929, + "primary_contact_id": 66378659, + "priority": "None", + "status": "Open", + "tags": [], + "interaction_count": 32, + "monetary_unit": null, + "monetary_value": null, + "converted_unit": null, + "converted_value": null, + "win_probability": 0, + "date_stage_changed": 1539985501, + "date_last_contacted": 1544830560, + "leads_converted_from": [], + "date_lead_created": null, + "date_created": 1538416597, + "date_modified": 1544830709, + "custom_fields": [ + { "custom_field_definition_id": 261066, "value": [394016] }, + { "custom_field_definition_id": 261067, "value": 394023 } + ] + }, + { + "id": 14209841, + "name": "Blockfolio TW", + "assignee_id": 680302, + "close_date": "11/15/2018", + "company_id": 29332516, + "company_name": "Blockfolio", + "customer_source_id": null, + "details": "blah blah", + "loss_reason_id": null, + "pipeline_id": 512676, + "pipeline_stage_id": 2405443, + "primary_contact_id": 67247027, + "priority": "None", + "status": "Open", + "tags": [], + "interaction_count": 20, + "monetary_unit": null, + "monetary_value": null, + "converted_unit": null, + "converted_value": null, + "win_probability": 0, + "date_stage_changed": 1539984098, + "date_last_contacted": 1539977661, + "leads_converted_from": [], + "date_lead_created": null, + "date_created": 1539624801, + "date_modified": 1539984098, + "custom_fields": [ + { "custom_field_definition_id": 261066, "value": [394015] }, + { "custom_field_definition_id": 261067, "value": 394026 } + ] + }, + { + "id": 14633220, + "name": "BlockSwap 721 / 1155 Conversational Marketplace", + "assignee_id": 680302, + "close_date": "12/15/2018", + "company_id": 30210921, + "company_name": "BlockSwap", + "customer_source_id": null, + "details": "blah blah", + "loss_reason_id": null, + "pipeline_id": 512676, + "pipeline_stage_id": 2392929, + "primary_contact_id": 69296220, + "priority": "None", + "status": "Open", + "tags": [], + "interaction_count": 82, + "monetary_unit": null, + "monetary_value": null, + "converted_unit": null, + "converted_value": null, + "win_probability": 0, + "date_stage_changed": 1542311056, + "date_last_contacted": 1543536442, + "leads_converted_from": [], + "date_lead_created": null, + "date_created": 1542311056, + "date_modified": 1543557877, + "custom_fields": [ + { "custom_field_definition_id": 261066, "value": [394014] }, + { "custom_field_definition_id": 261067, "value": 394023 } + ] + } +] diff --git a/packages/pipeline/test/fixtures/copper/api_v1_list_opportunities.ts b/packages/pipeline/test/fixtures/copper/api_v1_list_opportunities.ts new file mode 100644 index 000000000..3c2d4ae5e --- /dev/null +++ b/packages/pipeline/test/fixtures/copper/api_v1_list_opportunities.ts @@ -0,0 +1,425 @@ +// tslint:disable:custom-no-magic-numbers +import { CopperOpportunity } from '../../../src/entities'; +const ParsedOpportunities: CopperOpportunity[] = [ + { + id: 14050269, + name: '8Base RaaS', + assigneeId: 680302, + closeDate: '11/19/2018', + companyId: 27778962, + companyName: '8base', + customerSourceId: undefined, + lossReasonId: undefined, + pipelineId: 512676, + pipelineStageId: 2405442, + primaryContactId: 66088850, + priority: 'None', + status: 'Won', + interactionCount: 81, + monetaryValue: undefined, + winProbability: 0, + dateCreated: 1538414159000, + dateModified: 1544769562000, + customFields: { '261066': 394018, '261067': 394026 }, + }, + { + id: 14631430, + name: 'Alice.si TW + ERC 20 Marketplace', + assigneeId: 680302, + closeDate: '12/15/2018', + companyId: 30238847, + companyName: 'Alice SI', + customerSourceId: undefined, + lossReasonId: undefined, + pipelineId: 512676, + pipelineStageId: 2392929, + primaryContactId: 69354024, + priority: 'None', + status: 'Open', + interactionCount: 4, + monetaryValue: undefined, + winProbability: 0, + dateCreated: 1542304481000, + dateModified: 1542304943000, + customFields: { '261066': 394015, '261067': 394023 }, + }, + { + id: 14632057, + name: 'Altcoin.io Relayer', + assigneeId: 680302, + closeDate: '12/15/2018', + companyId: 29936486, + companyName: 'Altcoin.io', + customerSourceId: undefined, + lossReasonId: undefined, + pipelineId: 512676, + pipelineStageId: 2392929, + primaryContactId: 68724646, + priority: 'None', + status: 'Open', + interactionCount: 22, + monetaryValue: undefined, + winProbability: 0, + dateCreated: 1542306827000, + dateModified: 1543864667000, + customFields: { '261066': 394017, '261067': 394023 }, + }, + { + id: 14667523, + name: 'Altcoin.io Relayer', + assigneeId: 680302, + closeDate: '12/19/2018', + companyId: 29936486, + companyName: 'Altcoin.io', + customerSourceId: undefined, + lossReasonId: undefined, + pipelineId: 512676, + pipelineStageId: 2392929, + primaryContactId: 68724646, + priority: 'None', + status: 'Open', + interactionCount: 21, + monetaryValue: undefined, + winProbability: 0, + dateCreated: 1542657437000, + dateModified: 1543864667000, + customFields: { '261066': 394017, '261067': 394023 }, + }, + { + id: 14666706, + name: 'Amadeus Relayer', + assigneeId: 680302, + closeDate: '11/19/2018', + companyId: 29243209, + companyName: 'Amadeus', + customerSourceId: undefined, + lossReasonId: undefined, + pipelineId: 512676, + pipelineStageId: 2405442, + primaryContactId: 66912020, + priority: 'None', + status: 'Won', + interactionCount: 11, + monetaryValue: undefined, + winProbability: 0, + dateCreated: 1542654284000, + dateModified: 1543277520000, + customFields: { '261066': 394013, '261067': 394023 }, + }, + { + id: 14666718, + name: 'Ambo Relayer', + assigneeId: 680302, + closeDate: '11/19/2018', + companyId: 29249190, + companyName: 'Ambo', + customerSourceId: undefined, + lossReasonId: undefined, + pipelineId: 512676, + pipelineStageId: 2405442, + primaryContactId: 66927869, + priority: 'None', + status: 'Won', + interactionCount: 126, + monetaryValue: undefined, + winProbability: 0, + dateCreated: 1542654352000, + dateModified: 1545253761000, + customFields: { '261066': 394013, '261067': 394023 }, + }, + { + id: 14164318, + name: 'Augur TW', + assigneeId: 680302, + closeDate: '12/10/2018', + companyId: 27778967, + companyName: 'Augur', + customerSourceId: undefined, + lossReasonId: undefined, + pipelineId: 512676, + pipelineStageId: 2405442, + primaryContactId: 67248692, + priority: 'None', + status: 'Won', + interactionCount: 22, + monetaryValue: undefined, + winProbability: 0, + dateCreated: 1539204858000, + dateModified: 1544653867000, + customFields: { '261066': 394015, '261067': 394021 }, + }, + { + id: 14666626, + name: 'Autonio', + assigneeId: 680302, + closeDate: '12/19/2018', + companyId: 27920701, + companyName: 'Auton', + customerSourceId: undefined, + lossReasonId: undefined, + pipelineId: 512676, + pipelineStageId: 2392931, + primaryContactId: 64742640, + priority: 'None', + status: 'Open', + interactionCount: 54, + monetaryValue: undefined, + winProbability: 0, + dateCreated: 1542653834000, + dateModified: 1542658808000, + customFields: { '261066': 394019, '261067': 394023 }, + }, + { + id: 14050921, + name: 'Axie Infinity 721 Marketplace', + assigneeId: 680302, + closeDate: '11/1/2018', + companyId: 27779033, + companyName: 'Axie Infinity', + customerSourceId: undefined, + lossReasonId: undefined, + pipelineId: 512676, + pipelineStageId: 2392931, + primaryContactId: 66499254, + priority: 'None', + status: 'Open', + interactionCount: 4, + monetaryValue: undefined, + winProbability: 0, + dateCreated: 1538416687000, + dateModified: 1543861025000, + customFields: { '261066': 394014, '261067': 394134 }, + }, + { + id: 13735617, + name: 'Balance TW', + assigneeId: 680302, + closeDate: '12/10/2018', + companyId: 27778968, + companyName: 'Balance', + customerSourceId: undefined, + lossReasonId: undefined, + pipelineId: 512676, + pipelineStageId: 2405442, + primaryContactId: 64713448, + priority: 'None', + status: 'Won', + interactionCount: 34, + monetaryValue: undefined, + winProbability: 0, + dateCreated: 1535668009000, + dateModified: 1545082454000, + customFields: { '261066': 394015, '261067': 394027 }, + }, + { + id: 14667112, + name: 'Bamboo Relayer', + assigneeId: 680302, + closeDate: '11/19/2018', + companyId: 29243795, + companyName: 'Bamboo Relay', + customerSourceId: undefined, + lossReasonId: undefined, + pipelineId: 512676, + pipelineStageId: 2405442, + primaryContactId: 66914687, + priority: 'None', + status: 'Won', + interactionCount: 46, + monetaryValue: undefined, + winProbability: 0, + dateCreated: 1542655143000, + dateModified: 1545253761000, + customFields: { '261066': 394013, '261067': 394023 }, + }, + { + id: 13627309, + name: 'Ben TW', + assigneeId: 680302, + closeDate: '1/1/2019', + companyId: 27702348, + companyName: 'Ben', + customerSourceId: undefined, + lossReasonId: undefined, + pipelineId: 512676, + pipelineStageId: 2392929, + primaryContactId: 64262622, + priority: 'None', + status: 'Open', + interactionCount: 64, + monetaryValue: undefined, + winProbability: 0, + dateCreated: 1534887789000, + dateModified: 1541651395000, + customFields: { '261066': 394015, '261067': 394027 }, + }, + { + id: 14808512, + name: 'Bit2Me Relayer', + assigneeId: 680302, + closeDate: '12/3/2018', + companyId: 30793050, + companyName: 'Bit2Me', + customerSourceId: undefined, + lossReasonId: undefined, + pipelineId: 512676, + pipelineStageId: 2405442, + primaryContactId: 70267217, + priority: 'None', + status: 'Won', + interactionCount: 0, + monetaryValue: undefined, + winProbability: 0, + dateCreated: 1543861167000, + dateModified: 1543861189000, + customFields: { '261066': 394013, '261067': 394023 }, + }, + { + id: 14050312, + name: 'Bitcoin.tax Reporting Integration', + assigneeId: 680302, + closeDate: '11/1/2018', + companyId: 27957614, + companyName: 'Bitcoin', + customerSourceId: undefined, + lossReasonId: undefined, + pipelineId: 512676, + pipelineStageId: 2392928, + primaryContactId: 66539479, + priority: 'None', + status: 'Open', + interactionCount: 5, + monetaryValue: undefined, + winProbability: 0, + dateCreated: 1538414308000, + dateModified: 1538414314000, + customFields: { '261066': 394019, '261067': 394026 }, + }, + { + id: 14331463, + name: 'Bitpie TW', + assigneeId: 680302, + closeDate: '11/19/2018', + companyId: 27779026, + companyName: 'Bitpie', + customerSourceId: undefined, + lossReasonId: undefined, + pipelineId: 512676, + pipelineStageId: 2392929, + primaryContactId: 67700943, + priority: 'None', + status: 'Open', + interactionCount: 9, + monetaryValue: undefined, + winProbability: 0, + dateCreated: 1539984566000, + dateModified: 1541530233000, + customFields: { '261066': 394015, '261067': 394027 }, + }, + { + id: 14331481, + name: 'Bitski Wallet SDK TW', + assigneeId: 680302, + closeDate: '11/19/2018', + companyId: 29489300, + companyName: 'Bitski', + customerSourceId: undefined, + lossReasonId: undefined, + pipelineId: 512676, + pipelineStageId: 2392929, + primaryContactId: 67697528, + priority: 'None', + status: 'Open', + interactionCount: 23, + monetaryValue: undefined, + winProbability: 0, + dateCreated: 1539984735000, + dateModified: 1544818605000, + customFields: { '261066': 394015, '261067': 394026 }, + }, + { + id: 14531554, + name: 'BitUniverse TW', + assigneeId: 680302, + closeDate: '12/6/2018', + companyId: 29901805, + companyName: 'BitUniverse Co., Ltd (Cryptocurrency Portfolio)', + customerSourceId: undefined, + lossReasonId: undefined, + pipelineId: 512676, + pipelineStageId: 2392929, + primaryContactId: 68692107, + priority: 'None', + status: 'Open', + interactionCount: 15, + monetaryValue: undefined, + winProbability: 0, + dateCreated: 1541527110000, + dateModified: 1544812979000, + customFields: { '261066': 394015, '261067': 394026 }, + }, + { + id: 14050895, + name: 'BlitzPredict PMR', + assigneeId: 680302, + closeDate: '11/1/2018', + companyId: 28758258, + companyName: 'BlitzPredict', + customerSourceId: undefined, + lossReasonId: undefined, + pipelineId: 512676, + pipelineStageId: 2392929, + primaryContactId: 66378659, + priority: 'None', + status: 'Open', + interactionCount: 32, + monetaryValue: undefined, + winProbability: 0, + dateCreated: 1538416597000, + dateModified: 1544830709000, + customFields: { '261066': 394016, '261067': 394023 }, + }, + { + id: 14209841, + name: 'Blockfolio TW', + assigneeId: 680302, + closeDate: '11/15/2018', + companyId: 29332516, + companyName: 'Blockfolio', + customerSourceId: undefined, + lossReasonId: undefined, + pipelineId: 512676, + pipelineStageId: 2405443, + primaryContactId: 67247027, + priority: 'None', + status: 'Open', + interactionCount: 20, + monetaryValue: undefined, + winProbability: 0, + dateCreated: 1539624801000, + dateModified: 1539984098000, + customFields: { '261066': 394015, '261067': 394026 }, + }, + { + id: 14633220, + name: 'BlockSwap 721 / 1155 Conversational Marketplace', + assigneeId: 680302, + closeDate: '12/15/2018', + companyId: 30210921, + companyName: 'BlockSwap', + customerSourceId: undefined, + lossReasonId: undefined, + pipelineId: 512676, + pipelineStageId: 2392929, + primaryContactId: 69296220, + priority: 'None', + status: 'Open', + interactionCount: 82, + monetaryValue: undefined, + winProbability: 0, + dateCreated: 1542311056000, + dateModified: 1543557877000, + customFields: { '261066': 394014, '261067': 394023 }, + }, +]; +export { ParsedOpportunities }; diff --git a/packages/pipeline/test/fixtures/copper/parsed_entities.ts b/packages/pipeline/test/fixtures/copper/parsed_entities.ts new file mode 100644 index 000000000..1f49d38ed --- /dev/null +++ b/packages/pipeline/test/fixtures/copper/parsed_entities.ts @@ -0,0 +1,5 @@ +export { ParsedActivityTypes } from './api_v1_activity_types'; +export { ParsedCustomFields } from './api_v1_custom_field_definitions'; +export { ParsedActivities } from './api_v1_list_activities'; +export { ParsedLeads } from './api_v1_list_leads'; +export { ParsedOpportunities } from './api_v1_list_opportunities'; diff --git a/packages/pipeline/test/parsers/copper/index_test.ts b/packages/pipeline/test/parsers/copper/index_test.ts new file mode 100644 index 000000000..bb8e70da1 --- /dev/null +++ b/packages/pipeline/test/parsers/copper/index_test.ts @@ -0,0 +1,87 @@ +import * as chai from 'chai'; +import 'mocha'; + +import { + CopperActivity, + CopperActivityType, + CopperCustomField, + CopperLead, + CopperOpportunity, +} from '../../../src/entities'; +import { + CopperActivityResponse, + CopperActivityTypeCategory, + CopperActivityTypeResponse, + CopperCustomFieldResponse, + CopperSearchResponse, + parseActivities, + parseActivityTypes, + parseCustomFields, + parseLeads, + parseOpportunities, +} from '../../../src/parsers/copper'; +import { chaiSetup } from '../../utils/chai_setup'; + +chaiSetup.configure(); +const expect = chai.expect; + +type CopperResponse = CopperSearchResponse | CopperCustomFieldResponse; +type CopperEntity = CopperLead | CopperActivity | CopperOpportunity | CopperActivityType | CopperCustomField; + +import * as activityTypesApiResponse from '../../fixtures/copper/api_v1_activity_types.json'; +import * as customFieldsApiResponse from '../../fixtures/copper/api_v1_custom_field_definitions.json'; +import * as listActivitiesApiResponse from '../../fixtures/copper/api_v1_list_activities.json'; +import * as listLeadsApiResponse from '../../fixtures/copper/api_v1_list_leads.json'; +import * as listOpportunitiesApiResponse from '../../fixtures/copper/api_v1_list_opportunities.json'; +import { + ParsedActivities, + ParsedActivityTypes, + ParsedCustomFields, + ParsedLeads, + ParsedOpportunities, +} from '../../fixtures/copper/parsed_entities'; + +interface TestCase { + input: CopperResponse[]; + expected: CopperEntity[]; + parseFn(input: CopperResponse[]): CopperEntity[]; +} +const testCases: TestCase[] = [ + { + input: listLeadsApiResponse, + expected: ParsedLeads, + parseFn: parseLeads, + }, + { + input: (listActivitiesApiResponse as unknown) as CopperActivityResponse[], + expected: ParsedActivities, + parseFn: parseActivities, + }, + { + input: listOpportunitiesApiResponse, + expected: ParsedOpportunities, + parseFn: parseOpportunities, + }, + { + input: customFieldsApiResponse, + expected: ParsedCustomFields, + parseFn: parseCustomFields, + }, +]; +describe('Copper parser', () => { + it('parses API responses', () => { + testCases.forEach(testCase => { + const actual: CopperEntity[] = testCase.parseFn(testCase.input); + expect(actual).deep.equal(testCase.expected); + }); + }); + + // special case because the API response is not an array + it('parses activity types API response', () => { + const actual: CopperActivityType[] = parseActivityTypes((activityTypesApiResponse as unknown) as Map< + CopperActivityTypeCategory, + CopperActivityTypeResponse[] + >); + expect(actual).deep.equal(ParsedActivityTypes); + }); +}); diff --git a/packages/pipeline/tsconfig.json b/packages/pipeline/tsconfig.json index 6f138f260..45e07374c 100644 --- a/packages/pipeline/tsconfig.json +++ b/packages/pipeline/tsconfig.json @@ -4,7 +4,15 @@ "outDir": "lib", "rootDir": ".", "emitDecoratorMetadata": true, - "experimentalDecorators": true + "experimentalDecorators": true, + "resolveJsonModule": true }, - "include": ["./src/**/*", "./test/**/*", "./migrations/**/*"] + "include": ["./src/**/*", "./test/**/*", "./migrations/**/*"], + "files": [ + "./test/fixtures/copper/api_v1_activity_types.json", + "./test/fixtures/copper/api_v1_custom_field_definitions.json", + "./test/fixtures/copper/api_v1_list_activities.json", + "./test/fixtures/copper/api_v1_list_leads.json", + "./test/fixtures/copper/api_v1_list_opportunities.json" + ] } diff --git a/packages/website/translations/chinese.json b/packages/website/translations/chinese.json index b99a3cdcb..88193a181 100644 --- a/packages/website/translations/chinese.json +++ b/packages/website/translations/chinese.json @@ -83,7 +83,7 @@ "BUILD_A_RELAYER": "build a relayer", "BUILD_A_RELAYER_DESCRIPTION": "Learn how to build your own 0x relayer from scratch", "DEVELOP_ON_ETHEREUM": "develop on Ethereum", - "DEVELOP_ON_ETHEREUM_DESCRIPTION": "Learn more about building applications ontop of Ethereum", + "DEVELOP_ON_ETHEREUM_DESCRIPTION": "Learn more about building applications on top of Ethereum", "ORDER_BASICS": "Make & take orders", "ORDER_BASICS_DESCRIPTION": "Tutorial on how to create, validate and fill an order using 0x", "USE_NETWORKED_LIQUIDITY": "use networked liquidity", diff --git a/packages/website/translations/english.json b/packages/website/translations/english.json index 2914ffead..a1d8ecc43 100644 --- a/packages/website/translations/english.json +++ b/packages/website/translations/english.json @@ -87,7 +87,7 @@ "BUILD_A_RELAYER": "build a relayer", "BUILD_A_RELAYER_DESCRIPTION": "Learn how to build your own 0x relayer from scratch", "DEVELOP_ON_ETHEREUM": "develop on Ethereum", - "DEVELOP_ON_ETHEREUM_DESCRIPTION": "Learn more about building applications ontop of Ethereum", + "DEVELOP_ON_ETHEREUM_DESCRIPTION": "Learn more about building applications on top of Ethereum", "ORDER_BASICS": "Make & take orders", "ORDER_BASICS_DESCRIPTION": "Tutorial on how to create, validate and fill an order using 0x", "USE_NETWORKED_LIQUIDITY": "use networked liquidity", diff --git a/packages/website/translations/korean.json b/packages/website/translations/korean.json index a421ffb94..539b81470 100644 --- a/packages/website/translations/korean.json +++ b/packages/website/translations/korean.json @@ -83,7 +83,7 @@ "BUILD_A_RELAYER": "build a relayer", "BUILD_A_RELAYER_DESCRIPTION": "Learn how to build your own 0x relayer from scratch", "DEVELOP_ON_ETHEREUM": "develop on Ethereum", - "DEVELOP_ON_ETHEREUM_DESCRIPTION": "Learn more about building applications ontop of Ethereum", + "DEVELOP_ON_ETHEREUM_DESCRIPTION": "Learn more about building applications on top of Ethereum", "ORDER_BASICS": "Make & take orders", "ORDER_BASICS_DESCRIPTION": "Tutorial on how to create, validate and fill an order using 0x", "USE_NETWORKED_LIQUIDITY": "use networked liquidity", diff --git a/packages/website/translations/russian.json b/packages/website/translations/russian.json index b3ea29cf3..feb5df02e 100644 --- a/packages/website/translations/russian.json +++ b/packages/website/translations/russian.json @@ -83,7 +83,7 @@ "BUILD_A_RELAYER": "build a relayer", "BUILD_A_RELAYER_DESCRIPTION": "Learn how to build your own 0x relayer from scratch", "DEVELOP_ON_ETHEREUM": "develop on Ethereum", - "DEVELOP_ON_ETHEREUM_DESCRIPTION": "Learn more about building applications ontop of Ethereum", + "DEVELOP_ON_ETHEREUM_DESCRIPTION": "Learn more about building applications on top of Ethereum", "ORDER_BASICS": "Make & take orders", "ORDER_BASICS_DESCRIPTION": "Tutorial on how to create, validate and fill an order using 0x", "USE_NETWORKED_LIQUIDITY": "use networked liquidity", diff --git a/packages/website/translations/spanish.json b/packages/website/translations/spanish.json index db75312c5..f4762a67e 100644 --- a/packages/website/translations/spanish.json +++ b/packages/website/translations/spanish.json @@ -84,7 +84,7 @@ "BUILD_A_RELAYER": "build a relayer", "BUILD_A_RELAYER_DESCRIPTION": "Learn how to build your own 0x relayer from scratch", "DEVELOP_ON_ETHEREUM": "develop on Ethereum", - "DEVELOP_ON_ETHEREUM_DESCRIPTION": "Learn more about building applications ontop of Ethereum", + "DEVELOP_ON_ETHEREUM_DESCRIPTION": "Learn more about building applications on top of Ethereum", "ORDER_BASICS": "Make & take orders", "ORDER_BASICS_DESCRIPTION": "Tutorial on how to create, validate and fill an order using 0x", "USE_NETWORKED_LIQUIDITY": "use networked liquidity", diff --git a/python-packages/json_schemas/.discharge.json b/python-packages/json_schemas/.discharge.json new file mode 100644 index 000000000..66d95679f --- /dev/null +++ b/python-packages/json_schemas/.discharge.json @@ -0,0 +1,13 @@ +{ + "domain": "0x-json-schemas-py", + "build_command": "python setup.py build_sphinx", + "upload_directory": "build/docs/html", + "index_key": "index.html", + "error_key": "index.html", + "trailing_slashes": true, + "cache": 3600, + "aws_profile": "default", + "aws_region": "us-east-1", + "cdn": false, + "dns_configured": true +} diff --git a/python-packages/json_schemas/.pylintrc b/python-packages/json_schemas/.pylintrc new file mode 100644 index 000000000..937bc6313 --- /dev/null +++ b/python-packages/json_schemas/.pylintrc @@ -0,0 +1,3 @@ +[MESSAGES CONTROL] +disable=C0330,line-too-long,fixme,too-few-public-methods,too-many-ancestors +# C0330 is "bad hanging indent". we use indents per `black`. diff --git a/python-packages/json_schemas/README.md b/python-packages/json_schemas/README.md new file mode 100644 index 000000000..ef8e888f3 --- /dev/null +++ b/python-packages/json_schemas/README.md @@ -0,0 +1,45 @@ +## 0x-json-schemas + +0x JSON schemas for those developing on top of 0x protocol. + +Read the [documentation](http://0x-json-schemas-py.s3-website-us-east-1.amazonaws.com/) + +## Installing + +```bash +pip install 0x-json-schemas +``` + +## Contributing + +We welcome improvements and fixes from the wider community! To report bugs within this package, please create an issue in this repository. + +Please read our [contribution guidelines](../../CONTRIBUTING.md) before getting started. + +### Install Code and Dependencies + +Ensure that you have installed Python >=3.6 and Docker. Then: + +```bash +pip install -e .[dev] +``` + +### Test + +Tests depend on a running ganache instance with the 0x contracts deployed in it. For convenience, a docker container is provided that has ganache-cli and a snapshot containing the necessary contracts. A shortcut is provided to run that docker container: `./setup.py ganache`. With that running, the tests can be run with `./setup.py test`. + +### Clean + +`./setup.py clean --all` + +### Lint + +`./setup.py lint` + +### Build Documentation + +`./setup.py build_sphinx` + +### More + +See `./setup.py --help-commands` for more info. diff --git a/python-packages/json_schemas/setup.py b/python-packages/json_schemas/setup.py new file mode 100755 index 000000000..7813f101f --- /dev/null +++ b/python-packages/json_schemas/setup.py @@ -0,0 +1,191 @@ +#!/usr/bin/env python + +"""setuptools module for json_schemas package.""" + +import distutils.command.build_py +from distutils.command.clean import clean +import subprocess # nosec +from shutil import rmtree +from os import environ, path +from sys import argv + +from setuptools import find_packages, setup +from setuptools.command.test import test as TestCommand + + +class TestCommandExtension(TestCommand): + """Run pytest tests.""" + + def run_tests(self): + """Invoke pytest.""" + import pytest + + exit(pytest.main()) + + +class LintCommand(distutils.command.build_py.build_py): + """Custom setuptools command class for running linters.""" + + description = "Run linters" + + def run(self): + """Run linter shell commands.""" + lint_commands = [ + # formatter: + "black --line-length 79 --check --diff src test setup.py".split(), + # style guide checker (formerly pep8): + "pycodestyle src test setup.py".split(), + # docstring style checker: + "pydocstyle src test setup.py".split(), + # static type checker: + "mypy src test setup.py".split(), + # security issue checker: + "bandit -r src ./setup.py".split(), + # general linter: + "pylint src test setup.py".split(), + # pylint takes relatively long to run, so it runs last, to enable + # fast failures. + ] + + # tell mypy where to find interface stubs for 3rd party libs + environ["MYPYPATH"] = path.join( + path.dirname(path.realpath(argv[0])), "stubs" + ) + + for lint_command in lint_commands: + print( + "Running lint command `", " ".join(lint_command).strip(), "`" + ) + subprocess.check_call(lint_command) # nosec + + +class CleanCommandExtension(clean): + """Custom command to do custom cleanup.""" + + def run(self): + """Run the regular clean, followed by our custom commands.""" + super().run() + rmtree("dist", ignore_errors=True) + rmtree(".mypy_cache", ignore_errors=True) + rmtree(".tox", ignore_errors=True) + rmtree(".pytest_cache", ignore_errors=True) + rmtree("src/*.egg-info", ignore_errors=True) + + +class TestPublishCommand(distutils.command.build_py.build_py): + """Custom command to publish to test.pypi.org.""" + + description = ( + "Publish dist/* to test.pypi.org. Run sdist & bdist_wheel first." + ) + + def run(self): + """Run twine to upload to test.pypi.org.""" + subprocess.check_call( # nosec + ( + "twine upload --repository-url https://test.pypi.org/legacy/" + + " --verbose dist/*" + ).split() + ) + + +class PublishCommand(distutils.command.build_py.build_py): + """Custom command to publish to pypi.org.""" + + description = "Publish dist/* to pypi.org. Run sdist & bdist_wheel first." + + def run(self): + """Run twine to upload to pypi.org.""" + subprocess.check_call("twine upload dist/*".split()) # nosec + + +class PublishDocsCommand(distutils.command.build_py.build_py): + """Custom command to publish docs to S3.""" + + description = ( + "Publish docs to " + + "http://0x-json-schemas-py.s3-website-us-east-1.amazonaws.com/" + ) + + def run(self): + """Run npm package `discharge` to build & upload docs.""" + subprocess.check_call("discharge deploy".split()) # nosec + + +with open("README.md", "r") as file_handle: + README_MD = file_handle.read() + + +setup( + name="0x-json-schemas", + version="1.0.0", + description="JSON schemas for 0x applications", + long_description=README_MD, + long_description_content_type="text/markdown", + url=( + "https://github.com/0xProject/0x-monorepo/tree/development" + + "/python-packages/json_schemas" + ), + author="F. Eugene Aumson", + author_email="feuGeneA@users.noreply.github.com", + cmdclass={ + "clean": CleanCommandExtension, + "lint": LintCommand, + "test": TestCommandExtension, + "test_publish": TestPublishCommand, + "publish": PublishCommand, + "publish_docs": PublishDocsCommand, + }, + install_requires=["jsonschema", "mypy_extensions", "stringcase"], + extras_require={ + "dev": [ + "bandit", + "black", + "coverage", + "coveralls", + "mypy", + "mypy_extensions", + "pycodestyle", + "pydocstyle", + "pylint", + "pytest", + "sphinx", + "tox", + "twine", + ] + }, + python_requires=">=3.6, <4", + package_data={"zero_ex.json_schemas": ["py.typed", "schemas/*"]}, + package_dir={"": "src"}, + license="Apache 2.0", + keywords=( + "ethereum cryptocurrency 0x decentralized blockchain dex exchange" + ), + namespace_packages=["zero_ex"], + packages=find_packages("src"), + classifiers=[ + "Development Status :: 2 - Pre-Alpha", + "Intended Audience :: Developers", + "Intended Audience :: Financial and Insurance Industry", + "License :: OSI Approved :: Apache Software License", + "Natural Language :: English", + "Operating System :: OS Independent", + "Programming Language :: Python", + "Programming Language :: Python :: 3 :: Only", + "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.7", + "Topic :: Internet :: WWW/HTTP", + "Topic :: Office/Business :: Financial", + "Topic :: Other/Nonlisted Topic", + "Topic :: Security :: Cryptography", + "Topic :: Software Development :: Libraries", + "Topic :: Utilities", + ], + zip_safe=False, # required per mypy + command_options={ + "build_sphinx": { + "source_dir": ("setup.py", "src"), + "build_dir": ("setup.py", "build/docs"), + } + }, +) diff --git a/python-packages/json_schemas/src/conf.py b/python-packages/json_schemas/src/conf.py new file mode 100644 index 000000000..1ae1493e3 --- /dev/null +++ b/python-packages/json_schemas/src/conf.py @@ -0,0 +1,54 @@ +"""Configuration file for the Sphinx documentation builder.""" + +# Reference: http://www.sphinx-doc.org/en/master/config + +from typing import List +import pkg_resources + + +# pylint: disable=invalid-name +# because these variables are not named in upper case, as globals should be. + +project = "0x-json-schemas" +# pylint: disable=redefined-builtin +copyright = "2018, ZeroEx, Intl." +author = "F. Eugene Aumson" +version = pkg_resources.get_distribution("0x-json-schemas").version +release = "" # The full version, including alpha/beta/rc tags + +extensions = [ + "sphinx.ext.autodoc", + "sphinx.ext.doctest", + "sphinx.ext.intersphinx", + "sphinx.ext.coverage", + "sphinx.ext.viewcode", +] + +templates_path = ["doc_templates"] + +source_suffix = ".rst" +# eg: source_suffix = [".rst", ".md"] + +master_doc = "index" # The master toctree document. + +language = None + +exclude_patterns: List[str] = [] + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = None + +html_theme = "alabaster" + +html_static_path = ["doc_static"] +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". + +# Output file base name for HTML help builder. +htmlhelp_basename = "json_schemaspydoc" + +# -- Extension configuration: + +# Example configuration for intersphinx: refer to the Python standard library. +intersphinx_mapping = {"https://docs.python.org/": None} diff --git a/python-packages/order_utils/stubs/jsonschema/exceptions.pyi b/python-packages/json_schemas/src/doc_static/.gitkeep index e69de29bb..e69de29bb 100644 --- a/python-packages/order_utils/stubs/jsonschema/exceptions.pyi +++ b/python-packages/json_schemas/src/doc_static/.gitkeep diff --git a/python-packages/json_schemas/src/index.rst b/python-packages/json_schemas/src/index.rst new file mode 100644 index 000000000..3de809aa3 --- /dev/null +++ b/python-packages/json_schemas/src/index.rst @@ -0,0 +1,18 @@ +.. source for the sphinx-generated build/docs/web/index.html + +Python zero_ex.json_schemas +=========================== + +.. toctree:: + :maxdepth: 2 + :caption: Contents: + +.. automodule:: zero_ex.json_schemas + :members: + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` diff --git a/python-packages/json_schemas/src/zero_ex/__init__.py b/python-packages/json_schemas/src/zero_ex/__init__.py new file mode 100644 index 000000000..e90d833db --- /dev/null +++ b/python-packages/json_schemas/src/zero_ex/__init__.py @@ -0,0 +1,2 @@ +"""0x Python API.""" +__import__("pkg_resources").declare_namespace(__name__) diff --git a/python-packages/order_utils/src/zero_ex/json_schemas/__init__.py b/python-packages/json_schemas/src/zero_ex/json_schemas/__init__.py index a76a2fa3b..10c564b99 100644 --- a/python-packages/order_utils/src/zero_ex/json_schemas/__init__.py +++ b/python-packages/json_schemas/src/zero_ex/json_schemas/__init__.py @@ -6,6 +6,7 @@ from typing import Mapping from pkg_resources import resource_string import jsonschema +from stringcase import snakecase class _LocalRefResolver(jsonschema.RefResolver): @@ -13,20 +14,10 @@ class _LocalRefResolver(jsonschema.RefResolver): def __init__(self): """Initialize a new instance.""" - self.ref_to_file = { - "/addressSchema": "address_schema.json", - "/hexSchema": "hex_schema.json", - "/orderSchema": "order_schema.json", - "/wholeNumberSchema": "whole_number_schema.json", - "/ECSignature": "ec_signature_schema.json", - "/signedOrderSchema": "signed_order_schema.json", - "/ecSignatureParameterSchema": ( - "ec_signature_parameter_schema.json" + "" - ), - } jsonschema.RefResolver.__init__(self, "", "") - def resolve_from_url(self, url: str) -> str: + @staticmethod + def resolve_from_url(url: str) -> str: """Resolve the given URL. :param url: a string representing the URL of the JSON schema to fetch. @@ -35,15 +26,11 @@ class _LocalRefResolver(jsonschema.RefResolver): `url` does not exist. """ ref = url.replace("file://", "") - if ref in self.ref_to_file: - return json.loads( - resource_string( - "zero_ex.json_schemas", f"schemas/{self.ref_to_file[ref]}" - ) + return json.loads( + resource_string( + "zero_ex.json_schemas", + f"schemas/{snakecase(ref.lstrip('/'))}.json", ) - raise jsonschema.ValidationError( - f"Unknown ref '{ref}'. " - + f"Known refs: {list(self.ref_to_file.keys())}." ) @@ -65,10 +52,32 @@ def assert_valid(data: Mapping, schema_id: str) -> None: >>> assert_valid( ... {'v': 27, 'r': '0x'+'f'*64, 's': '0x'+'f'*64}, - ... '/ECSignature', + ... '/ecSignatureSchema', ... ) """ # noqa _, schema = _LOCAL_RESOLVER.resolve(schema_id) jsonschema.validate(data, schema, resolver=_LOCAL_RESOLVER) + + +def assert_valid_json(data: str, schema_id: str) -> None: + """Validate the given `data` against the specified `schema`. + + :param data: JSON string to be validated. + :param schema_id: id property of the JSON schema to validate against. Must + be one of those listed in `the 0x JSON schema files + <https://github.com/0xProject/0x-monorepo/tree/development/packages/json-schemas/schemas>`_. + + Raises an exception if validation fails. + + >>> assert_valid_json( + ... r'''{ + ... "v": 27, + ... "r": "0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", + ... "s": "0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff" + ... }''', + ... '/ecSignatureSchema', + ... ) + """ # noqa: E501 (line too long) + assert_valid(json.loads(data), schema_id) diff --git a/python-packages/json_schemas/src/zero_ex/json_schemas/py.typed b/python-packages/json_schemas/src/zero_ex/json_schemas/py.typed new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/python-packages/json_schemas/src/zero_ex/json_schemas/py.typed diff --git a/python-packages/order_utils/src/zero_ex/json_schemas/schemas b/python-packages/json_schemas/src/zero_ex/json_schemas/schemas index b8257372c..b8257372c 120000 --- a/python-packages/order_utils/src/zero_ex/json_schemas/schemas +++ b/python-packages/json_schemas/src/zero_ex/json_schemas/schemas diff --git a/python-packages/json_schemas/stubs/distutils/__init__.pyi b/python-packages/json_schemas/stubs/distutils/__init__.pyi new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/python-packages/json_schemas/stubs/distutils/__init__.pyi diff --git a/python-packages/json_schemas/stubs/distutils/command/__init__.pyi b/python-packages/json_schemas/stubs/distutils/command/__init__.pyi new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/python-packages/json_schemas/stubs/distutils/command/__init__.pyi diff --git a/python-packages/json_schemas/stubs/distutils/command/clean.pyi b/python-packages/json_schemas/stubs/distutils/command/clean.pyi new file mode 100644 index 000000000..46a42ddb1 --- /dev/null +++ b/python-packages/json_schemas/stubs/distutils/command/clean.pyi @@ -0,0 +1,7 @@ +from distutils.core import Command + +class clean(Command): + def initialize_options(self: clean) -> None: ... + def finalize_options(self: clean) -> None: ... + def run(self: clean) -> None: ... + ... diff --git a/python-packages/order_utils/stubs/jsonschema/__init__.pyi b/python-packages/json_schemas/stubs/jsonschema/__init__.pyi index 442e2f65e..442e2f65e 100644 --- a/python-packages/order_utils/stubs/jsonschema/__init__.pyi +++ b/python-packages/json_schemas/stubs/jsonschema/__init__.pyi diff --git a/python-packages/json_schemas/stubs/jsonschema/exceptions.pyi b/python-packages/json_schemas/stubs/jsonschema/exceptions.pyi new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/python-packages/json_schemas/stubs/jsonschema/exceptions.pyi diff --git a/python-packages/json_schemas/stubs/pytest/__init__.pyi b/python-packages/json_schemas/stubs/pytest/__init__.pyi new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/python-packages/json_schemas/stubs/pytest/__init__.pyi diff --git a/python-packages/json_schemas/stubs/pytest/raises.pyi b/python-packages/json_schemas/stubs/pytest/raises.pyi new file mode 100644 index 000000000..2e3b29f3d --- /dev/null +++ b/python-packages/json_schemas/stubs/pytest/raises.pyi @@ -0,0 +1 @@ +def raises(exception: Exception) -> ExceptionInfo: ... diff --git a/python-packages/json_schemas/stubs/setuptools/__init__.pyi b/python-packages/json_schemas/stubs/setuptools/__init__.pyi new file mode 100644 index 000000000..8ea8d32b7 --- /dev/null +++ b/python-packages/json_schemas/stubs/setuptools/__init__.pyi @@ -0,0 +1,8 @@ +from distutils.dist import Distribution +from typing import Any, List + +def setup(**attrs: Any) -> Distribution: ... + +class Command: ... + +def find_packages(where: str) -> List[str]: ... diff --git a/python-packages/json_schemas/stubs/setuptools/command/__init__.pyi b/python-packages/json_schemas/stubs/setuptools/command/__init__.pyi new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/python-packages/json_schemas/stubs/setuptools/command/__init__.pyi diff --git a/python-packages/json_schemas/stubs/setuptools/command/test.pyi b/python-packages/json_schemas/stubs/setuptools/command/test.pyi new file mode 100644 index 000000000..c5ec770ad --- /dev/null +++ b/python-packages/json_schemas/stubs/setuptools/command/test.pyi @@ -0,0 +1,3 @@ +from setuptools import Command + +class test(Command): ... diff --git a/python-packages/json_schemas/stubs/stringcase/__init__.pyi b/python-packages/json_schemas/stubs/stringcase/__init__.pyi new file mode 100644 index 000000000..56d784cf5 --- /dev/null +++ b/python-packages/json_schemas/stubs/stringcase/__init__.pyi @@ -0,0 +1,2 @@ +def snakecase(_: str): + ... diff --git a/python-packages/json_schemas/test/__init__.py b/python-packages/json_schemas/test/__init__.py new file mode 100644 index 000000000..ce724e180 --- /dev/null +++ b/python-packages/json_schemas/test/__init__.py @@ -0,0 +1 @@ +"""Tests of zero_ex.json_schemas.""" diff --git a/python-packages/json_schemas/test/test_doctest.py b/python-packages/json_schemas/test/test_doctest.py new file mode 100644 index 000000000..2aa576422 --- /dev/null +++ b/python-packages/json_schemas/test/test_doctest.py @@ -0,0 +1,25 @@ +"""Exercise doctests for all of our modules.""" + +from doctest import testmod +import pkgutil +import importlib + +import zero_ex.json_schemas + + +def test_all_doctests(): + """Gather zero_ex.json_schemas.* modules and doctest them.""" + # json_schemas module + module = "zero_ex.json_schemas" + print(module) + failure_count, _ = testmod(importlib.import_module(module)) + assert failure_count == 0 + + # any json_schemas.* sub-modules + for (_, modname, _) in pkgutil.walk_packages( + path=zero_ex.json_schemas.__path__, prefix="zero_ex.json_schemas" + ): + module = importlib.import_module(modname) + print(module) + (failure_count, _) = testmod(module) + assert failure_count == 0 diff --git a/python-packages/order_utils/test/test_json_schemas.py b/python-packages/json_schemas/test/test_json_schemas.py index 51cecbd4f..d0e9840b3 100644 --- a/python-packages/order_utils/test/test_json_schemas.py +++ b/python-packages/json_schemas/test/test_json_schemas.py @@ -1,10 +1,28 @@ """Tests of zero_ex.json_schemas""" -from zero_ex.order_utils import make_empty_order from zero_ex.json_schemas import _LOCAL_RESOLVER, assert_valid +NULL_ADDRESS = "0x0000000000000000000000000000000000000000" + +EMPTY_ORDER = { + "makerAddress": NULL_ADDRESS, + "takerAddress": NULL_ADDRESS, + "senderAddress": NULL_ADDRESS, + "feeRecipientAddress": NULL_ADDRESS, + "makerAssetData": NULL_ADDRESS, + "takerAssetData": NULL_ADDRESS, + "salt": "0", + "makerFee": "0", + "takerFee": "0", + "makerAssetAmount": "0", + "takerAssetAmount": "0", + "expirationTimeSeconds": "0", + "exchangeAddress": NULL_ADDRESS, +} + + def test_assert_valid_caches_resources(): """Test that the JSON ref resolver in `assert_valid()` caches resources @@ -15,7 +33,7 @@ def test_assert_valid_caches_resources(): """ _LOCAL_RESOLVER._remote_cache.cache_clear() # pylint: disable=W0212 - assert_valid(make_empty_order(), "/orderSchema") + assert_valid(EMPTY_ORDER, "/orderSchema") cache_info = ( _LOCAL_RESOLVER._remote_cache.cache_info() # pylint: disable=W0212 ) diff --git a/python-packages/json_schemas/tox.ini b/python-packages/json_schemas/tox.ini new file mode 100644 index 000000000..1d5de646e --- /dev/null +++ b/python-packages/json_schemas/tox.ini @@ -0,0 +1,25 @@ +# tox (https://tox.readthedocs.io/) is a tool for running tests +# in multiple virtualenvs. This configuration file will run the +# test suite on all supported python versions. To use it, "pip install tox" +# and then run "tox" from this directory. + +[tox] +envlist = py37 + +[testenv] +commands = + pip install -e .[dev] + python setup.py test + +[testenv:run_tests_against_test_deployment] +commands = + # install dependencies from real PyPI + pip install jsonschema mypy_extensions pytest + # install package-under-test from test PyPI + pip install --index-url https://test.pypi.org/legacy/ 0x-json-schemas + pytest test + +[testenv:run_tests_against_deployment] +commands = + pip install 0x-json-schemas + pytest test diff --git a/python-packages/order_utils/setup.py b/python-packages/order_utils/setup.py index 125de5ff7..06533e60a 100755 --- a/python-packages/order_utils/setup.py +++ b/python-packages/order_utils/setup.py @@ -137,14 +137,8 @@ class GanacheCommand(distutils.command.build_py.build_py): def run(self): """Run ganache.""" cmd_line = ( - "docker run -d -p 8545:8545 0xorg/ganache-cli --gasLimit" - + " 10000000 --db /snapshot --noVMErrorsOnRPCResponse -p 8545" - + " --networkId 50 -m" + "docker run -d -p 8545:8545 0xorg/ganache-cli:2.2.2" ).split() - cmd_line.append( - "concert load couple harbor equip island argue ramp clarify fence" - + " smart topic" - ) subprocess.call(cmd_line) # nosec @@ -171,9 +165,9 @@ setup( "ganache": GanacheCommand, }, install_requires=[ + "0x-json-schemas", "eth-abi", "eth_utils", - "jsonschema", "mypy_extensions", "web3", ], @@ -198,7 +192,6 @@ setup( package_data={ "zero_ex.order_utils": ["py.typed"], "zero_ex.contract_artifacts": ["artifacts/*"], - "zero_ex.json_schemas": ["schemas/*"], }, package_dir={"": "src"}, license="Apache 2.0", diff --git a/python-packages/order_utils/src/index.rst b/python-packages/order_utils/src/index.rst index 551487ab1..4d27a4b17 100644 --- a/python-packages/order_utils/src/index.rst +++ b/python-packages/order_utils/src/index.rst @@ -22,9 +22,6 @@ See source for class properties. Sphinx does not easily generate class property .. autoclass:: zero_ex.order_utils.asset_data_utils.ERC721AssetData -.. automodule:: zero_ex.json_schemas - :members: - Indices and tables ================== diff --git a/python-packages/order_utils/stubs/web3/__init___BASE_31011.pyi b/python-packages/order_utils/stubs/web3/__init___BASE_31011.pyi new file mode 100644 index 000000000..fcecc7434 --- /dev/null +++ b/python-packages/order_utils/stubs/web3/__init___BASE_31011.pyi @@ -0,0 +1,26 @@ +from typing import Dict, Optional, Union + +from web3.utils import datatypes + + +class Web3: + class HTTPProvider: ... + + def __init__(self, provider: HTTPProvider) -> None: ... + + @staticmethod + def sha3( + primitive: Optional[Union[bytes, int, None]] = None, + text: Optional[str] = None, + hexstr: Optional[str] = None + ) -> bytes: ... + + class net: + version: str + ... + + class eth: + @staticmethod + def contract(address: str, abi: Dict) -> datatypes.Contract: ... + ... + ... diff --git a/python-packages/sra_client/README.md b/python-packages/sra_client/README.md index ab3939b41..279fb41a4 100644 --- a/python-packages/sra_client/README.md +++ b/python-packages/sra_client/README.md @@ -6,6 +6,35 @@ A Python client for interacting with servers conforming to [the Standard Relayer The [JSON schemas](http://json-schema.org/) for the API payloads and responses can be found in [@0xproject/json-schemas](https://github.com/0xProject/0x.js/tree/development/packages/json-schemas). Examples of each payload and response can be found in the 0x.js library's [test suite](https://github.com/0xProject/0x.js/blob/development/packages/json-schemas/test/schema_test.ts#L1). +```bash +pip install 0x-json-schemas +``` + +You can easily validate your API's payloads and responses using the [0x-json-schemas](https://github.com/0xProject/0x.js/tree/development/python-packages/json_schemas) package: + +```python +from zero_ex.json_schemas import assert_valid +from zero_ex.order_utils import Order + +order: Order = { + 'makerAddress': "0x0000000000000000000000000000000000000000", + 'takerAddress': "0x0000000000000000000000000000000000000000", + 'feeRecipientAddress': "0x0000000000000000000000000000000000000000", + 'senderAddress': "0x0000000000000000000000000000000000000000", + 'makerAssetAmount': "1000000000000000000", + 'takerAssetAmount': "1000000000000000000", + 'makerFee': "0", + 'takerFee': "0", + 'expirationTimeSeconds': "12345", + 'salt': "12345", + 'makerAssetData': "0x0000000000000000000000000000000000000000", + 'takerAssetData': "0x0000000000000000000000000000000000000000", + 'exchangeAddress': "0x0000000000000000000000000000000000000000", +} + +assert_valid(order, "/orderSchema") +``` + # Pagination Requests that return potentially large collections should respond to the **?page** and **?perPage** parameters. For example: |