Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

first pass at EC structure and placement #120

Draft
wants to merge 4 commits into
base: master
Choose a base branch
from
+359 −0
Draft

first pass at EC structure and placement #120

Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
4c23ff0
first pass at EC structure and placement
dryajov Aug 18, 2022
ac3cf52
move to correct dir and cleanup
dryajov Aug 18, 2022
8cbeb52
reworked to support state
dryajov Aug 19, 2022
faacc0b
allow appending blocks and nodes
dryajov Aug 20, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
359 changes: 359 additions & 0 deletions analysis/ec-placement.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,359 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": 56,
"metadata": {},
"outputs": [
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "0d654a31aa8842bea08f69d17e1e9283",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"VBox(children=(Label(value='EC Layout and Placement'), VBox(children=(IntSlider(value=2, description='K', max=…"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "577931878dbf419d85441e582a1760f8",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"Output()"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"from __future__ import print_function\n",
"from ipywidgets import interact, interactive, fixed, interact_manual\n",
"import ipywidgets as widgets\n",
"import itertools as it\n",
"import tabulate as tb\n",
"from functools import reduce\n",
"from operator import concat\n",
"from collections import OrderedDict\n",
"import math\n",
"\n",
"DEFAULT_BLOCKS = 12\n",
"DEFAULT_NODES = 3\n",
"DEFAULT_K = 2\n",
"DEFAULT_M = 1\n",
"DEFAULT_LAYOUT = \"Inline\"\n",
"DEFAULT_DIRECTION = \"Horizontal\"\n",
"\n",
"\n",
"def roundBlocks(factor, ob):\n",
" return (factor - (ob % factor)) if (ob % factor) > 0 else 0\n",
"\n",
"\n",
"def makeEcGroups(\n",
" blocks,\n",
" k, m,\n",
" direction,\n",
" offset=0):\n",
"\n",
" offset = len(blocks) if offset == 0 else offset\n",
" res = []\n",
"\n",
" groups = (len(blocks) // k)\n",
" steps = len(blocks) // groups\n",
" if direction == \"Horizontal\":\n",
" for b in range(0, len(blocks), steps):\n",
" res.append(\n",
" blocks[b:b + steps] +\n",
" list(range(offset + 1, (offset + 1 + m))))\n",
" offset += m\n",
" else:\n",
" for s in range(0, groups):\n",
" res.append(\n",
" blocks[s:s+groups+offset:groups] +\n",
" list(range(offset + 1, (offset + 1 + m))))\n",
" offset += m\n",
"\n",
" return res\n",
"\n",
"\n",
"def mapNodes(blocks, hosts, offset=0):\n",
" width = getZeros(len(blocks) or BC.value)\n",
" formatBlock = getBlockFormat(width)\n",
" res = OrderedDict()\n",
"\n",
" for i, _ in enumerate(blocks):\n",
" key = \"node\" + str((i % hosts)+1)\n",
" res[key] = (key in res and res[key]) or []\n",
" res[key].append(formatBlock(blocks[i]))\n",
"\n",
" return res\n",
"\n",
"\n",
"def getBlockFormat(width):\n",
" return lambda b: \"{b:0{fmt}}\".format(b=b, fmt=width)\n",
"\n",
"\n",
"def getZeros(width):\n",
" return int(math.log(width, 10) + 1)\n",
"\n",
"\n",
"def makeReadSeq(K, M, ecGroups, direction, layout):\n",
" readSequence = []\n",
" if layout == \"Append\":\n",
" if direction == \"Horizontal\":\n",
" readSequence = reduce(\n",
" concat,\n",
" [g[:K] for g in ecGroups] + [g[-M:] for g in ecGroups])\n",
" else:\n",
" readSequence = reduce(concat, zip(*ecGroups))\n",
" else:\n",
" readSequence = reduce(concat, ecGroups)\n",
"\n",
" return readSequence\n",
"\n",
"\n",
"class ECPlacement:\n",
" blocks = []\n",
" nodesMap = OrderedDict()\n",
" blockSeq = []\n",
" ecGroups = []\n",
" rb = 0\n",
"\n",
" def __init__(\n",
" self,\n",
" k=DEFAULT_K, m=DEFAULT_M,\n",
" layout=DEFAULT_LAYOUT,\n",
" direction=DEFAULT_LAYOUT,\n",
" nodes=DEFAULT_NODES,\n",
" bc=DEFAULT_BLOCKS):\n",
" self.blocks = list(range(1, bc + 1))\n",
" self.k = k\n",
" self.m = m\n",
" self.layout = layout\n",
" self.direction = direction\n",
" self.nodes = nodes\n",
" self.bc = bc\n",
" self.blocks = []\n",
" self.nodesMap = OrderedDict()\n",
" self.blockSeq = []\n",
" self.ecGroups = []\n",
" self.rb = 0\n",
"\n",
" def reset(self):\n",
" self.k = DEFAULT_K\n",
" self.m = DEFAULT_M\n",
" self.layout = DEFAULT_LAYOUT\n",
" self.direction = DEFAULT_DIRECTION\n",
" self.nodes = DEFAULT_NODES\n",
" self.bc = DEFAULT_BLOCKS\n",
" self.rb = 0\n",
"\n",
" self.blocks = list(range(1, self.bc + 1))\n",
" self.nodesMap = OrderedDict()\n",
" self.blockSeq = []\n",
" self.ecGroups = []\n",
"\n",
" def update(self, k, m, bc, direction, layout, nodes, append):\n",
" print(k, m, bc, direction, layout, nodes, append)\n",
"\n",
" if append:\n",
" if bc > self.bc:\n",
" appendBlocks = bc - self.bc\n",
" appendBlocks = appendBlocks + roundBlocks(k, appendBlocks)\n",
" self.blocks += list(\n",
" range(self.bc + 1, bc + appendBlocks + 1))\n",
"\n",
" if len(self.blocks) > 0:\n",
" blockSeqLen = len(self.blockSeq)\n",
" print(blockSeqLen)\n",
"\n",
" newBlocks = list(\n",
" range(blockSeqLen + 1, blockSeqLen + appendBlocks + 1))\n",
"\n",
" self.ecGroups += makeEcGroups(\n",
" newBlocks, k, m, direction, blockSeqLen + appendBlocks) # Calculate EC groups\n",
"\n",
" self.blockSeq = makeReadSeq(\n",
" k, m, self.ecGroups, direction, layout)\n",
"\n",
" self.nodesMap = mapNodes(\n",
" self.blockSeq, nodes).items()\n",
" else:\n",
" # Calculate geometry\n",
" self.rb = roundBlocks(k, bc)\n",
" # Generate source blocks\n",
" self.blocks = list(\n",
" range(1, bc + 1 + self.rb))\n",
"\n",
" if len(self.blocks) > 0:\n",
" self.ecGroups = makeEcGroups(\n",
" self.blocks, k, m, direction) # Calculate EC groups\n",
"\n",
" self.blockSeq = makeReadSeq(\n",
" k, m, self.ecGroups, direction, layout)\n",
"\n",
" self.nodesMap = mapNodes(\n",
" self.blockSeq, nodes).items()\n",
"\n",
" self.bc = bc\n",
" self.k = k\n",
" self.m = m\n",
" self.direction = dir\n",
" self.layout = layout\n",
" self.nodes = nodes\n",
"\n",
" self.render()\n",
"\n",
" def render(self):\n",
" width = getZeros(len(self.blockSeq) or self.bc)\n",
" formatBlock = getBlockFormat(width)\n",
"\n",
" print(\"\\n\")\n",
" print(\"Rounded Blocks:\", self.rb)\n",
" print(\"Encoding Groups:\", len(self.ecGroups))\n",
" print(\"K + M = \", self.k + self.m)\n",
" print(\"Source Blocks:\", \", \".join(map(formatBlock, self.blocks)))\n",
" print(\"Read Layout:\", \", \".join(map(formatBlock, self.blockSeq)))\n",
"\n",
" groupsTable = tb.tabulate(\n",
" [[\", \".join(map(formatBlock, g[:self.k])), \", \".join(map(formatBlock, g[-self.m:]))]\n",
" for g in self.ecGroups], [\"Source\", \"Parity\"])\n",
" print(\"\\n\", groupsTable)\n",
"\n",
" nodesTable = tb.tabulate(\n",
" list(map(list, self.nodesMap)), [\"Nodes\", \"Blocks\"])\n",
" print(\"\\n\", nodesTable)\n",
"\n",
"\n",
"ecPlacement = ECPlacement()\n",
"\n",
"\n",
"def init(k, m, bc, dir, layout, nodes, append):\n",
" # K - The number of source blocks to encode\n",
" # BC - Initial block count\n",
" # Dir - Coding direction\n",
" # Layout - Block layout\n",
" # Nodes - Nodes count\n",
" ##\n",
"\n",
" ecPlacement.update(k, m, bc, dir, layout, nodes, append)\n",
"\n",
"\n",
"MAX_BLOCKS = 256\n",
"K = widgets.IntSlider(description=\"K\", min=2, max=MAX_BLOCKS)\n",
"M = widgets.IntSlider(description=\"M\", min=1, max=MAX_BLOCKS)\n",
"BC = widgets.IntSlider(description=\"Origina blocks count\",\n",
" min=1, max=10000, value=DEFAULT_BLOCKS)\n",
"\n",
"D = widgets.Dropdown(options=['Horizontal', 'Vertical'],\n",
" description=\"EC Coding Direction\")\n",
"\n",
"BlocksLayout = widgets.Dropdown(\n",
" options=['Inline', 'Append'],\n",
" description=\"Blocks Manifest Layout\")\n",
"\n",
"Clear = widgets.Button(description='Clear')\n",
"Nodes = widgets.IntSlider(\n",
" description=\"Nodes\", min=K.value + M.value, max=MAX_BLOCKS)\n",
"\n",
"Append = widgets.Checkbox(\n",
" value=False,\n",
" description='Append',\n",
" disabled=False,\n",
" indent=False)\n",
"\n",
"ui = widgets.VBox([\n",
" widgets.Label(value=\"EC Layout and Placement\"),\n",
" widgets.VBox([K, M, BC, D, BlocksLayout, Nodes, Append]),\n",
" Clear])\n",
"\n",
"\n",
"def clearModelClick(button):\n",
" ecPlacement.reset()\n",
" K.value = K.min\n",
" M.value = M.min\n",
" Nodes.value = Nodes.min = K.value + M.value\n",
" BC.value = 12\n",
" D.value = \"Horizontal\"\n",
" BlocksLayout.value = \"Inline\"\n",
" appendBlocks.value = 0\n",
" appendNodes = 0\n",
" Append.value = False\n",
"\n",
"\n",
"Clear.on_click(clearModelClick)\n",
"\n",
"\n",
"def updateKRange(*args):\n",
" K.max = MAX_BLOCKS - M.value\n",
" Nodes.value = K.value + M.value\n",
"\n",
"\n",
"M.observe(updateKRange, 'value')\n",
"\n",
"\n",
"def updateMRange(*args):\n",
" M.max = MAX_BLOCKS - K.value\n",
" Nodes.value = K.value + M.value\n",
"\n",
"\n",
"K.observe(updateMRange, 'value')\n",
"\n",
"\n",
"def updateNodesRange(*args):\n",
" Nodes.min = K.value + M.value\n",
"\n",
"\n",
"Nodes.observe(updateNodesRange, 'value')\n",
"\n",
"init = widgets.interactive_output(\n",
" init,\n",
" {\n",
" 'k': K,\n",
" 'm': M,\n",
" 'bc': BC,\n",
" 'dir': D,\n",
" 'layout': BlocksLayout,\n",
" 'nodes': Nodes,\n",
" 'append': Append})\n",
"\n",
"display(ui, init)\n"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3.9.5 64-bit ('3.9.5')",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.9.5"
},
"orig_nbformat": 4,
"vscode": {
"interpreter": {
"hash": "f341111545ab039e59db4a6307f8d37d8b30a2d09f0bbfe377cad911d3b9e229"
}
}
},
"nbformat": 4,
"nbformat_minor": 2
}