Skip to content

Commit

Permalink
(feat): background sync | sense 🧬 (#1083)
Browse files Browse the repository at this point in the history
Co-authored-by: dasein <[email protected]>
Co-authored-by: dimakorzhovnik <[email protected]>
  • Loading branch information
3 people authored Mar 11, 2024
1 parent 5b1e2f0 commit d91d373
Show file tree
Hide file tree
Showing 299 changed files with 13,536 additions and 2,137 deletions.
7 changes: 4 additions & 3 deletions .eslintrc
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,8 @@
],
"globals": {
"expect": true,
"FileReader": true
"FileReader": true,
"cyblog": "readonly"
},
"rules": {
"valid-jsdoc": "off",
Expand Down Expand Up @@ -125,12 +126,12 @@
"always",
{ "ts": "never", "tsx": "never", "jsx": "never", "js": "never" }
],
"import/no-unresolved": "off",
"no-unused-expressions": [
"error",
{ "allowShortCircuit": true, "allowTernary": true }
],
"no-underscore-dangle": "off"
"no-underscore-dangle": "off",
"import/no-unresolved": "off"
},
"settings": {
"import/resolver": {
Expand Down
11 changes: 11 additions & 0 deletions .storybook/main.ts
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,17 @@ const config: StorybookConfig = {
),
...config.plugins,
],

module: {
...config.module,
rules: [
...config.module.rules,
{
test: /\.cozo$/,
use: 'raw-loader',
},
],
},
};
},
docs: {
Expand Down
11 changes: 7 additions & 4 deletions .storybook/preview.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,9 @@ import store from '../src/redux/store';
import { Provider } from 'react-redux';
window.React = React;

// reuse from main app somehow
window.cyb = {};

export const queryClient = new QueryClient({
defaultOptions: {
queries: {
Expand Down Expand Up @@ -42,13 +45,13 @@ const preview: Preview = {
<div style={{ margin: '3em' }}>
<BrowserRouter>
<Provider store={store}>
<SdkQueryClientProvider>
<QueryClientProvider client={queryClient}>
<QueryClientProvider client={queryClient}>
<SdkQueryClientProvider>
<IbcDenomProvider>
<Story />
</IbcDenomProvider>
</QueryClientProvider>
</SdkQueryClientProvider>
</SdkQueryClientProvider>
</QueryClientProvider>
</Provider>
</BrowserRouter>
</div>
Expand Down
30 changes: 28 additions & 2 deletions .storybook/stubs.tsx
Original file line number Diff line number Diff line change
@@ -1,9 +1,35 @@
import { Coin } from '@cosmjs/launchpad';

export const title = 'Moon Citizenship';

export const text50 =
'Lorem ipsum, dolor sit amet consectetur adipisicing elit. Atque consectetur sint deleniti recusandae eveniet necessitatibus perferendis voluptate iusto molestias. Iure est suscipit amet laudantium illo expedita dolores possimus tempora, repellat modi deleniti sequi optio consequatur aperiam? Sit perspiciatis odit eos iure quo magnam, quia ea cumque, iste odio officiis ex!';

export const text100 =
'Welcome to the portal. Take a quick look on what Bostrom are, and what you can do here. Get or manage your Moon Citizenship, check the ongoing gift status and more. Welcome to the portal. Take a quick look on what Bostrom are, and what you can do here. Get or manage your Moon Citizenship, check the ongoing gift status and more. Welcome to the portal. Take a quick look on what Bostrom are, and what you can do here. Get or manage your Moon Citizenship';
'Lorem ipsum dolor sit amet consectetur adipisicing elit. Quo minus, culpa vitae, nostrum impedit nemo architecto dignissimos eius modi at iste ratione! Illum ad temporibus tempore est! Asperiores dolorem earum commodi delectus, nisi distinctio sit alias, omnis numquam dicta velit rerum quas facere pariatur autem aperiam eius laborum quo expedita voluptates totam sequi quibusdam voluptate? Perspiciatis corrupti molestias provident hic, nostrum nulla saepe repellendus veritatis tempora assumenda nobis in inventore quidem quas facilis impedit fugiat, cum asperiores dolorum non? Animi repellat laudantium ut vero unde enim quasi? Nostrum excepturi veniam eaque dignissimos beatae dicta, reiciendis, voluptatum animi sequi necessitatibus eos.';

export const text500 =
'Welcome to the portal. Take a quick look on what Bostrom are, and what you can do here. Get or manage your Moon Citizenship, check the ongoing gift status and more. Welcome to the portal. Take a quick look on what Bostrom are, and what you can do here. Get or manage your Moon Citizenship, check the ongoing gift status and more. Welcome to the portal. Take a quick look on what Bostrom are, and what you can do here. Get or manage your Moon Citizenship. Welcome to the portal. Take a quick look on what Bostrom are, and what you can do here. Get or manage your Moon Citizenship, check the ongoing gift status and more. Welcome to the portal. Take a quick look on what Bostrom are, and what you can do here. Get or manage your Moon Citizenship, check the ongoing gift status and more. Welcome to the portal. Take a quick look on what Bostrom are, and what you can do here. Get or manage your Moon Citizenship';
'Welcome to the portal. Take a quick look on what Bostrom are, and ≈what you can do here. Get or manage your Moon Citizenship, check the ongoing gift status and more. Welcome to the portal. Take a quick look on what Bostrom are, and what you can do here. Get or manage your Moon Citizenship, check the ongoing gift status and more. Welcome to the portal. Take a quick look on what Bostrom are, and what you can do here. Get or manage your Moon Citizenship. Welcome to the portal. Take a quick look on what Bostrom are, and what you can do here. Get or manage your Moon Citizenship, check the ongoing gift status and more. Welcome to the portal. Take a quick look on what Bostrom are, and what you can do here. Get or manage your Moon Citizenship, check the ongoing gift status and more. Welcome to the portal. Take a quick look on what Bostrom are, and what you can do here. Get or manage your Moon Citizenship';

export const img = 'https://cyb.ai/images/preview.png';

export const address = 'bostrom1f5warat4vc0q98k7ygys4saka8u04rfxpmthvl';
export const transactionHash =
'BAA06424541F011A759CE61DDA9D8A5471D87FFA93BAF7E82FA0FC02014D86E5';

export const particle = 'QmZ4b5kbCV9K9Jd2ZXpfUAKiPSkzPfD558wBbebYFpztKY';

export const amounts = [
{
denom: 'boot',
amount: '100000000',
},
{
denom: 'hydrogen',
amount: '100000000000',
},
{
denom: 'eth',
amount: '1.5',
},
] as Coin[];
30 changes: 19 additions & 11 deletions docs/backend.md
Original file line number Diff line number Diff line change
Expand Up @@ -6,26 +6,35 @@ graph TD;
subgraph frontend["frontend(main thread)"]
App["Frontend"]-->Hook["useBackend()"];
Hook-->methods("startSync()\nloadIpfs()\n...\nisReady\nipfsError\n...");
Hook--odbApi[/"dbApi(proxy)"/];
Hook--obackendApiFront[/"backendApi(proxy)"/];
Hook-.broadcast channel\n(any worker).->reducer["redux(state)"]
Hook-.save history from app.->defferedDbApiFront[/"DefferedDbApi(proxy)"/]
Hook--osenseApi["senseApi"];
Hook--oipfsApiFront[/"ipfsApi(proxy)"/];
senseApi--odbApi[/"dbApi(proxy)"/];
end
dbApi<-.message channel.->dbWorker["dbApi"];
subgraph dbWorkerGraph["cyb~db(worker)"]
dbWorker<-.bindings(webApi).->cozodb{{"CozoDb(wasm)"}}
end
backendApiFront<-.message channel.->backendApi;
defferedDbApiFront-.->defferedDbApi;
ipfsApiFront<-.->ipfsApi;
subgraph backgroundWorker["cyb~backend(worker)"]
backendApi--oipfsApi;
backendApi--oimporterApi;
importerApi;
ipfsApi;
importerApi-->ipfsApi;
subgraph sync["sync service"]
ipfsNode["ipfs node"];
links;
transactions;
end
sync--oparticleResolver[["Particle resolver"]]
particleResolver--oqueue;
particleResolver--odbProxyWorker;
sync--oipfsApi;
sync--odbProxyWorker[/"dbApi(proxy)"/];
defferedDbApi[["defferedDbApi"]]-->dbProxyWorker;
queue-->defferedDbApi;
ipfsApi--oqueue[["queue"]];
ipfsApi--o node["node"];
ipfsApi--onode["node"];
queue--balancer-->node;
node--embedded-->helia;
node--rpc-->kubo;
Expand All @@ -36,7 +45,6 @@ graph TD;
js-ipfs;
end
importerApi--odbProxyWorker[/"dbApi(proxy)"/];
dbProxyWorker<-.message channel.->dbWorker
end
Expand Down
71 changes: 71 additions & 0 deletions docs/sync.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,71 @@
## General

- every process is RxJs async loop with N-second interval
- all CID from's from any links in any process goes to separate _particle resolver queue(non-blocking)_

## My transactions process

Sync all current neuron transactions: extract tweets, and chat(any interractions with other neurons).

_transactions can be heavy, for ex. there can be MultiSend/broadcast transactions of 150-200Kb_

- save all transactions to DB
- create sense chats for all unique neurons from MsgSend/MsgMultisend
- extract 'tweet' links and add it to sync items(SyncParticlesLoop)
-
> **request count** = 1 aggregate req. + [user_transactions_count/batch].
> example - 1000 trans., batch is 200, requests = 1 + 1000/200 = 6
> every 60 sec.


```mermaid
flowchart TD
A[SyncTransactionLoop] --> P1
P1[get last timestamp] --> B
B(get trans. count > timestamp) <-..->|messages_by_address_aggregate| I[(Indexer)]
B --> C{has new?}
C -->|yes| D(fetch batch transactions)
D <-..->|messages_by_address 'batch=200'| I
D -->|iter by batch| IP{batch processor}
IP --> D
IP -->|has items| E{transaction type}
IP -->|complete| U(update last timestamp)
E -->|MsgSend/MsgMultiSend| T1(add neuron to sense chat)
E -->|Cyberlink 'tweet'| T2(add tweet to 'sync')
```

## My particles(tweets) process
Sync current user particle updates(new links)

> **request count** = user_links_from='tweet' \* [links_count/batch]
> example - 100 tweets. requests = \* [???/10] = [100...???]
> every 60 sec.
```mermaid
flowchart TD
A[SyncParticlesLoop] -->|interval| P1
P1[get user tweets with last timestamp] --> L
L[tweet processor] -->|iter by every tweet| F
F[fetch links > timestamp] --> L
F <-..->|Cyberlinks| I[(Indexer)]
```

## My friends process
Sync current user updates from friends(tweets&follows)

> **request count** = following*users * (1(indexer) + [2(lcd)])
> example - following is 10, requests = 10 \* (1 + [2]) = [10...30]
> every 60 sec.
```mermaid
flowchart TD
A[SyncMyFriends] -->|interval| L
L[my friends processor]<-->|iter by every friend| F
F(fetch count)<--> L
F <-..->|cyberlinks_aggregate 'tweets', 'follow'| I[(Indexer)]
F --> C{has new follows/tweets?}
C -->|yes| D(fetch links)
D <-..->|fetch 'tweets' > timestamp| LCD[(LCD)]
D <-..->|fetch 'follow' > timestamp| LCD[(LCD)]
```
1 change: 1 addition & 0 deletions jest.config.js
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ module.exports = {
'^src(.*)$': '<rootDir>/src/$1',
'^rune$': '<rootDir>/rune_build',
'#(.*)': '<rootDir>/node_modules/$1',
'^uint8arrays/(.*)$': '<rootDir>/node_modules/uint8arrays/dist/src/$1',
},
preset: 'ts-jest',
resolver: 'ts-jest-resolver',
Expand Down
22 changes: 8 additions & 14 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,7 @@
"@types/dateformat": "^5.0.0",
"@types/react-helmet": "^6.1.6",
"@types/react-transition-group": "^4.4.5",
"@types/uuid": "^9.0.7",
"@types/webpack": "^5.28.1",
"@typescript-eslint/eslint-plugin": "^5.57.0",
"@typescript-eslint/parser": "^5.57.0",
Expand Down Expand Up @@ -86,7 +87,7 @@
"https-browserify": "^1.0.0",
"husky": "^4.3.8",
"it-all": "3.0.1",
"jest": "^29.5.0",
"jest": "^29.7.0",
"jest-environment-jsdom": "^29.5.0",
"mini-css-extract-plugin": "^2.6.1",
"ncp": "^2.0.0",
Expand Down Expand Up @@ -134,7 +135,7 @@
"webpack-merge": "^5.7.3"
},
"dependencies": {
"@apollo/react-hooks": "^3.1.3",
"@apollo/client": "^3.9.4",
"@chainsafe/libp2p-noise": "^13.0.1",
"@chainsafe/libp2p-yamux": "^5.0.0",
"@confio/relayer": "0.4.0",
Expand Down Expand Up @@ -164,18 +165,12 @@
"@tanstack/react-table": "^8.9.1",
"@types/file-saver": "^2.0.5",
"@types/history": "^5.0.0",
"@types/jest": "^29.5.5",
"@types/jest": "^29.5.11",
"@types/ramda": "^0.28.23",
"@types/react-dom": "^18.0.11",
"@types/react-router-dom": "^5.3.3",
"@uniswap/sdk": "^3.0.3",
"apollo-boost": "^0.4.7",
"apollo-cache-inmemory": "^1.6.5",
"apollo-client": "^2.6.8",
"apollo-link": "^1.2.13",
"apollo-link-http": "^1.5.16",
"apollo-link-ws": "^1.0.19",
"apollo-utilities": "^1.3.3",
"bech32": "^1.1.3",
"big.js": "^5.2.2",
"bignumber.js": "^9.0.0",
Expand All @@ -186,7 +181,7 @@
"comlink": "^4.4.1",
"core-js": "^3.30.0",
"crypto": "^1.0.1",
"cyb-cozo-lib-wasm": "0.7.141",
"cyb-cozo-lib-wasm": "^0.7.145",
"datastore-core": "^9.2.3",
"datastore-idb": "^2.1.4",
"dateformat": "^3.0.3",
Expand All @@ -202,7 +197,7 @@
"file-type": "^18.2.1",
"graphql": "^16.5.0",
"graphql-request": "^5.1.0",
"graphql-tag": "^2.10.1",
"graphql-ws": "^5.15.0",
"helia": "^2.0.3",
"ipfs-core": "^0.18.0",
"ipfs-core-types": "^0.14.0",
Expand All @@ -215,6 +210,7 @@
"kubo-rpc-client": "^3.0.1",
"ledger-cosmos-js": "2.1.7",
"libp2p": "^0.46.12",
"lodash": "^4.17.21",
"long": "^5.2.0",
"node-polyfill-webpack-plugin": "^2.0.1",
"process": "^0.11.10",
Expand All @@ -223,7 +219,6 @@
"raw-loader": "^4.0.2",
"rc-slider": "^9.7.2",
"react": "^18.0.0",
"react-apollo": "^3.1.3",
"react-dom": "^18.0.0",
"react-force-graph": "^1.39.5",
"react-helmet": "^6.1.0",
Expand All @@ -243,9 +238,8 @@
"rxjs": "^7.8.0",
"secp256k1": "^3.7.1",
"sinon": "^16.0.0",
"subscriptions-transport-ws": "^0.9.18",
"tone": "^14.7.77",
"ts-jest": "^29.1.0",
"ts-jest": "^29.1.1",
"ts-jest-resolver": "^2.0.1",
"videostream": "^3.2.2",
"web3": "1.2.4",
Expand Down
14 changes: 14 additions & 0 deletions src/components/HydrogenBalance/HydrogenBalance.tsx
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
import { useGetBalanceBostrom } from 'src/containers/sigma/hooks';
import IconsNumber from '../IconsNumber/IconsNumber';

function HydrogenBalance({ address }) {
const { totalAmountInLiquid } = useGetBalanceBostrom(address);

return (
<div>
<IconsNumber value={totalAmountInLiquid.currentCap} type="hydrogen" />
</div>
);
}

export default HydrogenBalance;
2 changes: 2 additions & 0 deletions src/components/Input/Input.module.scss
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,8 @@
height: unset;

textarea {
position: relative;
top: -5px;
color: white;
resize: none;
box-shadow: unset;
Expand Down
4 changes: 4 additions & 0 deletions src/components/Input/InputNumber/InputNumber.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,8 @@ export type Props = {
title?: string;
color?: Color;
fixedDecimalScale?: boolean;
maxValue?: number;
min?: number;
};

function InputNumber({
Expand All @@ -22,6 +24,7 @@ function InputNumber({
fixedDecimalScale,
onChange,
maxValue,
min,
...props
}: Props) {
const [focused, setFocused] = useState(false);
Expand All @@ -44,6 +47,7 @@ function InputNumber({
return true;
}}
maxValue={maxValue}
min={min}
thousandsGroupStyle="thousand"
thousandSeparator=" "
decimalScale={3}
Expand Down
2 changes: 1 addition & 1 deletion src/components/MusicalAddress/utils.js
Original file line number Diff line number Diff line change
@@ -1,13 +1,13 @@
/* eslint-disable no-restricted-syntax */
import * as Tone from 'tone';
import {
PATTERN_CYBER,
PATTERN_ETH,
PATTERN_COSMOS,
PATTERN_OSMOS,
PATTERN_TERRA,
PATTERN_CYBER_VALOPER,
} from '../../utils/config';
import { PATTERN_CYBER } from 'src/constants/app';

const DICTIONARY_ABC = {
a: { note: 'E3', height: 16, gain: 1, color: '#36D6AE' },
Expand Down
Loading

0 comments on commit d91d373

Please sign in to comment.