mouse-and-keyboard-encoding-fft.ipynb

{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.10","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import torch\n\n# mouse -> sparse encoding -> fft -> ifft -> unified decoder\n\ndim_0_range = 1000\ndim_1_range = 100\n\nmouse_coords = [(20,20,None,None), (200,200,30,30)]","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-05-04T06:15:07.324467Z","iopub.execute_input":"2023-05-04T06:15:07.324854Z","iopub.status.idle":"2023-05-04T06:15:11.310032Z","shell.execute_reply.started":"2023-05-04T06:15:07.324822Z","shell.execute_reply":"2023-05-04T06:15:11.308915Z"},"trusted":true},"execution_count":1,"outputs":[]},{"cell_type":"code","source":"# what about sparse encoding?\n\n# single value -> bunch of binary values\n# elementwise product random vector -> select non-zero ones\n\nimport random\n\nrandom.seed(42)\n\nwindow_size = 200 \n\nmrange = list(range(window_size+dim_0_range-1))\n\nrandom.shuffle(mrange)\n\n# mlist = [mrange[i:i+window_size] for i in range(dim_0_range)]\n\n# keep it sparse?\n\nunified_encoding = torch.randn((1,window_size+dim_0_range-1), requires_grad=True)\n\n# that's how you initialize your \"semantic\" or \"continual\" mouse embeddings.\n\nmlist = []\n\nnext_comb = mrange[:window_size]\n# random.shuffle(next_comb)\n\nmlist.append(next_comb.copy())\n\nfor i in range(dim_0_range-1):\n#     last_item = mrange[i+window_size-1]\n    alt_item = mrange[i+window_size]\n    last_item_index = random.choice(range(window_size))\n    next_comb[last_item_index] = alt_item\n#     print(torch.Tensor([next_comb]))\n#     print(\"SUM?\", sum(next_comb))\n    mlist.append(next_comb.copy())\n\n# import rich\nprint(\"LENGTH?\", len(mlist))\n# rich.print(\"MLIST?\", mlist)\n\n# for e in mlist:\n#     print(sum(e))\n\nmLongTensorList = torch.LongTensor(mlist) # that looks like the thing.\nmLongTensorList.shape","metadata":{"execution":{"iopub.status.busy":"2023-05-04T06:15:11.314262Z","iopub.execute_input":"2023-05-04T06:15:11.314840Z","iopub.status.idle":"2023-05-04T06:15:11.383862Z","shell.execute_reply.started":"2023-05-04T06:15:11.314806Z","shell.execute_reply":"2023-05-04T06:15:11.382754Z"},"trusted":true},"execution_count":2,"outputs":[{"name":"stdout","text":"LENGTH? 1000\n","output_type":"stream"},{"execution_count":2,"output_type":"execute_result","data":{"text/plain":"torch.Size([1000, 200])"},"metadata":{}}]},{"cell_type":"code","source":"torch.index_select(unified_encoding, 1, mLongTensorList[0,:])","metadata":{"execution":{"iopub.status.busy":"2023-05-04T06:15:11.385515Z","iopub.execute_input":"2023-05-04T06:15:11.385838Z","iopub.status.idle":"2023-05-04T06:15:11.492349Z","shell.execute_reply.started":"2023-05-04T06:15:11.385810Z","shell.execute_reply":"2023-05-04T06:15:11.491070Z"},"trusted":true},"execution_count":3,"outputs":[{"execution_count":3,"output_type":"execute_result","data":{"text/plain":"tensor([[ 0.0201, -0.2096, -0.8161, -0.7944, -0.0273,  1.0396, -0.7215, -0.7736,\n          0.1442, -0.0986, -1.0944, -0.9450,  1.5013,  1.5580,  0.1765,  0.8375,\n          0.7912,  1.2963, -0.2231, -1.7506,  1.2640,  0.4092,  1.6101,  0.9551,\n          0.7136, -1.1262, -0.1486,  0.2570, -0.0386, -0.2204,  1.1235, -1.4930,\n          0.0791, -0.8223,  0.6044, -0.4391,  0.4782,  1.1106,  1.3433,  0.6720,\n          1.5228, -1.0026, -1.4263,  0.6802, -0.3787,  1.0678, -1.1386, -1.1958,\n          1.0507, -0.1179, -0.6561, -0.1105,  0.1971,  0.5198,  0.0422, -0.2031,\n          0.0433,  0.0375, -0.3140, -0.4879, -1.0317, -0.2368,  0.2479, -1.6437,\n         -0.9109, -0.2596,  0.3943, -0.5662,  0.7805, -0.4219,  0.2423,  0.0221,\n          0.6220,  0.0939, -0.2971,  1.0105,  2.6450,  0.1953,  0.1059,  0.6088,\n         -1.3216, -0.1247, -0.0680, -1.1350,  0.4346, -0.7606, -0.8015, -0.1266,\n          1.4999, -0.2279,  1.1375, -0.4531,  0.5546, -1.8871, -0.0098, -1.1060,\n         -2.0904,  0.9246, -1.1352, -0.3978, -0.5662,  0.2215, -0.4089,  1.6541,\n          0.0875,  0.5778,  0.3227, -0.3334,  1.1854,  0.9953,  1.4872,  0.9542,\n          0.7748, -0.2235,  0.2317, -0.8324, -0.2328,  0.2116, -1.3525,  0.1017,\n         -1.6839,  0.4476,  0.1996,  0.3367,  0.7257,  1.1923,  0.5673, -0.3300,\n          0.7330,  1.0731,  1.3492, -0.7070,  0.3828, -0.4624,  0.5125, -0.3975,\n          0.2134,  0.3259,  0.1678, -2.3008, -0.2759,  1.8816, -1.2616, -0.1667,\n         -0.9047, -0.9393, -1.8838, -1.4357, -0.3968, -0.7846,  0.1522, -1.0923,\n          0.5983, -0.2093,  0.5104, -0.1287,  0.5029, -0.8182,  1.0785,  0.4553,\n          0.3271,  2.0989,  1.6431,  0.7072,  0.2722, -0.3144,  0.6949, -2.3610,\n          0.0179, -0.7871, -1.1320, -0.4742, -0.0938, -1.3726,  1.9498,  0.6327,\n         -0.5436, -0.1136, -0.6661,  1.4590, -0.5336, -2.0114, -0.2179,  0.4321,\n          0.9994, -0.0954, -0.2132, -0.3729, -0.1824,  0.2083,  0.8734, -0.1582,\n          0.1618,  0.3165, -0.0144, -0.2303, -1.1744,  1.1025,  0.1774,  1.1039]],\n       grad_fn=<IndexSelectBackward0>)"},"metadata":{}}]},{"cell_type":"code","source":"# how to represent keyboard keydown signals?\n# telephone?\n\n# embedding plus one trainable sin keydown signal? or using fft?\n\n# how to represent special tokens? by sin? all by sin?\n\n# what will happen if you try to share vector space in non-standard way?\n# such as split and concat?\n\n# you may do split and concat in fft though.\n\n# such as: value repr by concat -> ifft -> LSTM -> fft -> argmax things\n\n# ifft let the model \"feel\" the bits, though fft \"extract\" freq and handle bits.\n\n# there are multiple ways to do this.\n\n# but fft brings \"imaginary\" part to numbers.\n# you can feed both parts separetely into the network, then combine different parts.\n# the you may calculate the grad? by adding different part of the loss?\n\n# or you use \"native\" complex neural networks, to handle the fft transforms.\n# or you simply ignore complex input. only taking real parts?\n\n# lstm contains hidden state and cell state\n# while GRU only contains hidden state.\n\n# adding real and imag? or passing through different NN? or same NN?\n# telling you, do it first. we will handle the comparison.\n\n# so are you going to tell me that my model is just complete\n# that i need not to do too much to collect data and start training?\n\n# yes. i am going to tell you to start training.\n# you have reached the utopia of fourier transform (rfft/hfft).\n# now let's roll!","metadata":{"execution":{"iopub.status.busy":"2023-05-04T06:15:11.494371Z","iopub.execute_input":"2023-05-04T06:15:11.494734Z","iopub.status.idle":"2023-05-04T06:15:11.500262Z","shell.execute_reply.started":"2023-05-04T06:15:11.494703Z","shell.execute_reply":"2023-05-04T06:15:11.498993Z"},"trusted":true},"execution_count":4,"outputs":[]},{"cell_type":"code","source":"# for ubuntu arm: pyautogui -> python script writing timestamp\n# write to stdout -> pipe to ffmpeg -> write to video\n\n# find the location of the shared directory of utm\n\n# how do i know if my model is spitting words instead of images/actions?\n# do we need to take over few \"positional encodings\"?\n\n# you can add task specific embeddings with token embedding\n# then decode the task type in the end, classify the token.\n\n# fft may not be needed, since that will be too much computation.\n# you may just want low rank adaption over some linear layers.\n\n# fft may be useful for your visual convolution.","metadata":{},"execution_count":null,"outputs":[]}]}