v0.8 release

yeokm1 · Apr 9, 2023 · c5b3c32 · c5b3c32
1 parent 32a57e5
commit c5b3c32
Show file tree

Hide file tree

Showing 14 changed files with 311 additions and 68 deletions.
diff --git a/README.md b/README.md
@@ -124,6 +124,9 @@ curl --proxy "http://192.168.1.144:8080" https://api.openai.com/v1/chat/completi
 ```
 
 # Changelog
+* v0.8 (9 Apr 2023):
+* * Supports Greek [Code Page 737](https://en.wikipedia.org/wiki/Code_page_437) via `-cp737` command line argument.
+* * Corrected small bug in Code Page 437 parsing UTF-8 characters starting with 0xE2 that does not return designated unknown character if unknown character is encountered.
 * v0.7 (8 Apr 2023):
 * * Corrected bug in previous release where previous message/reply memory is not freed after program ends.
 * * Now will use one-time malloc allocations of previous message (5000), temp message (5000), GPT reply (8000) buffers to avoid memory fragmentation.

diff --git a/api-data/acc-all-comma.txt → api-data/cp437.txt b/api-data/acc-all-comma.txt → api-data/cp437.txt
diff --git a/api-data/cp737-samples.txt b/api-data/cp737-samples.txt
@@ -0,0 +1 @@
+{"id":"chatcmpl-73JoAf0rMfuEnKjayuVg0BBqWb18O","object":"chat.completion","created":1681025482,"model":"gpt-3.5-turbo-0301","usage":{"prompt_tokens":23,"completion_tokens":234,"total_tokens":257},"choices":[{"message":{"role":"assistant","content":"As an AI language model, I cannot display characters in a visual format as they are shown in a computer. However, I can list the characters included in the Greek Code Page 737:Α,Β,Γ,Δ,Ε,Ζ,Η,Θ,Ι,Κ,Λ,Μ,Ν,Ξ,Ο,Π,Ρ,Σ,Τ,Υ,Φ,Χ,Ψ,Ω,α,β,γ,δ,ε,ζ,η,θ,ι,κ,λ,μ,ν,ξ,ο,π,ρ,σ,ς,τ,υ,φ,χ,ψ,░,▒,▓,│,┤,╡,╢,╖,╕,╣,║,╗,╝,╜,╛,┐,└,┴,┬,├,─,┼,╞,╟,╚,╔,╩,╦,╠,═,╬,╧,╨,╤,╥,╙,╘,╒,╓,╫,╪,┘,┌,█,▄,▌,▐,▀,ω,ά,έ,ή,ϊ,ί,ό,ύ,ϋ,ώ,Ά,Έ,Ή,Ί,Ό,Ύ,Ώ,±,≥,≤,Ϊ,Ϋ,÷,≈,°,∙,·,√,ⁿ,²,■"},"finish_reason":"stop","index":0}]}
diff --git a/api-data/reply.txt b/api-data/reply.txt
@@ -1 +1 @@
-{"id":"chatcmpl-703fFOKPE7ffRr0ecA4V9snJ5lPEG","object":"chat.completion","created":1680248441,"model":"gpt-3.5-turbo-0301","usage":{"prompt_tokens":20,"completion_tokens":610,"total_tokens":630},"choices":[{"message":{"role":"assistant","content":"Here is a list of extended ASCII characters from 128 to 255:,,128: Ç,129: ü,130: é,131: â,132: ä,133: à,134: å,135: ç,136: ê,137: ë,138: è,139: ï,140: î,141: ì,142: Ä,143: Å,144: É,145: æ,146: Æ,147: ô,148: ö,149: ò,150: û,151: ù,152: ÿ,153: Ö,154: Ü,155: ¢,156: £,157: ¥,158: ₧,159: ƒ,160: á,161: í,162: ó,163: ú,164: ñ,165: Ñ,166: ª,167: º,168: ¿,169: ⌐,170: ¬,171: ½,172: ¼,173: ¡,174: «,175: »,176: ░,177: ▒,178: ▓,179: │,180: ┤,181: ╡,182: ╢,183: ╖,184: ╕,185: ╣,186: ║,187: ╗,188: ╝,189: ╜,190: ╛,191: ┐,192: └,193: ┴,194: ┬,195: ├,196: ─,197: ┼,198: ╞,199: ╟,200: ╚,201: ╔,202: ╩,203: ╦,204: ╠,205: ═,206: ╬,207: ╧,208: ╨,209: ╤,210: ╥,211: ╙,212: ╘,213: ╒,214: ╓,215: ╫,216: ╪,217: ┘,218: ┌,219: █,220: ▄,221: ▌,222: ▐,223: ▀,224: α,225: ß,226: Γ,227: π,228: Σ,229: σ,230: µ,231: τ,232: Φ,233: Θ,234: Ω,235: δ,236: ∞,237: φ,238: ε,239: ∩,240: ≡,241: ±,242: ≥,243: ≤,244: ⌠,245: ⌡,246: ÷,247: ≈,248: °,249: ∙,250: ·,251: √,252: ⁿ,253: ²,254: ■,255:  "},"finish_reason":"stop","index":0}]}
+{"id":"chatcmpl-73JoAf0rMfuEnKjayuVg0BBqWb18O","object":"chat.completion","created":1681025482,"model":"gpt-3.5-turbo-0301","usage":{"prompt_tokens":23,"completion_tokens":234,"total_tokens":257},"choices":[{"message":{"role":"assistant","content":"As an AI language model, I cannot display characters in a visual format as they are shown in a computer. However, I can list the characters included in the Greek Code Page 737:Α,Β,Γ,Δ,Ε,Ζ,Η,Θ,Ι,Κ,Λ,Μ,Ν,Ξ,Ο,Π,Ρ,Σ,Τ,Υ,Φ,Χ,Ψ,Ω,α,β,γ,δ,ε,ζ,η,θ,ι,κ,λ,μ,ν,ξ,ο,π,ρ,σ,ς,τ,υ,φ,χ,ψ,░,▒,▓,│,┤,╡,╢,╖,╕,╣,║,╗,╝,╜,╛,┐,└,┴,┬,├,─,┼,╞,╟,╚,╔,╩,╦,╠,═,╬,╧,╨,╤,╥,╙,╘,╒,╓,╫,╪,┘,┌,█,▄,▌,▐,▀,ω,ά,έ,ή,ϊ,ί,ό,ύ,ϋ,ώ,Ά,Έ,Ή,Ί,Ό,Ύ,Ώ,±,≥,≤,Ϊ,Ϋ,÷,≈,°,∙,·,√,ⁿ,²,■"},"finish_reason":"stop","index":0}]}
diff --git a/doschgpt-code/MAKEFILE b/doschgpt-code/MAKEFILE
@@ -35,7 +35,7 @@ compile_options += -i=$(tcp_h_dir) -i=$(common_h_dir)
 
 
 tcpobjs = packet.obj arp.obj eth.obj ip.obj tcp.obj tcpsockm.obj udp.obj utils.obj dns.obj timer.obj ipasm.obj trace.obj
-objs = doschgpt.obj network.obj utfcp437.obj
+objs = doschgpt.obj network.obj utf2cp.obj utfcp437.obj utfcp737.obj
 
 all : clean doschgpt.exe
 

diff --git a/doschgpt-code/doschgpt.cpp b/doschgpt-code/doschgpt.cpp
@@ -4,9 +4,9 @@
 #include <string.h>
 
 #include "network.h"
-#include "utfcp437.h"
+#include "utf2cp.h"
 
-#define VERSION "0.7"
+#define VERSION "0.8"
 
 // User configuration
 #define CONFIG_FILENAME "doschgpt.ini"
@@ -28,6 +28,8 @@ uint16_t config_socketResponseTimeout;
 bool debug_showRequestInfo = false;
 bool debug_showRawReply = false;
 
+bool cp737_active = false;
+
 // Message Request
 #define SIZE_MSG_TO_SEND 4096
 char __far * messageToSendToNet;
@@ -137,6 +139,8 @@ int main(int argc, char * argv[]){
       debug_showRequestInfo = true;
     } else if(strstr(arg, "-drr")){
       debug_showRawReply = true;
+    } else if(strstr(arg, "-cp737")){
+      cp737_active = true;
     }
   }
 
@@ -155,6 +159,7 @@ int main(int argc, char * argv[]){
 
     printf("\nDebug request info -dri: %d\n", debug_showRequestInfo);
     printf("Debug raw reply -drr: %d\n", debug_showRawReply);
+    printf("Enable Greek CP737 -cp737: %d\n", cp737_active);
 
   } else {
     printf("\nCannot open %s config file containing:\nAPI key\nModel\nRequest Temperature\nProxy hostname\nProxy port\nOutgoing start port\nOutgoing end port\nSocket connect timeout (ms)\nSocket response timeout (ms)\n", CONFIG_FILENAME);
@@ -239,7 +244,14 @@ int main(int argc, char * argv[]){
               printf("\\");
               i++;
             } else {
-              CONVERSION_OUTPUT conversionResult = utf_to_cp437(currentChar, nextChar, followingChar);
+
+              CONVERSION_OUTPUT conversionResult;
+              if(cp737_active){
+                conversionResult = utf_to_cp(CODE_PAGE_737, currentChar, nextChar, followingChar);
+              } else {
+                conversionResult = utf_to_cp(CODE_PAGE_437, currentChar, nextChar, followingChar);
+              }
+
               unsigned char characterToPrint = conversionResult.character;
               int numCharactersToAdvance = conversionResult.charactersUsed - 1;
 

diff --git a/doschgpt-code/utf2cp.cpp b/doschgpt-code/utf2cp.cpp
@@ -0,0 +1,67 @@
+#include "utf2cp.h"
+#include "utfcp437.h"
+#include "utfcp737.h"
+
+CONVERSION_OUTPUT utf_to_cp(int codepage, int value0, int value1, int value2){
+    CONVERSION_OUTPUT output;
+
+    if(value0 <= 127){
+        output.character = value0;
+        output.charactersUsed = 1;
+        return output;
+    }
+    output.character = UNKNOWN_CHAR_REPLACEMENT;
+
+    switch(value0){
+        case 0xC2:
+            if(codepage == CODE_PAGE_737){
+                output.character = utf_to_cp737_c2(value1);
+            } else {
+                output.character = utf_to_cp437_c2(value1);
+            }
+            output.charactersUsed = 2;
+            break;
+        case 0xC3:
+            if(codepage == CODE_PAGE_737){
+                output.character = utf_to_cp737_c3(value1);
+            } else {
+                output.character = utf_to_cp437_c3(value1);
+            }
+            output.charactersUsed = 2;
+            break;
+        case 0xC6:
+            output.character = utf_to_cp437_c6(value1);
+            output.charactersUsed = 2;
+            break;
+        case 0xCE:
+            if(codepage == CODE_PAGE_737){
+                output.character = utf_to_cp737_ce(value1);
+            } else {
+                output.character = utf_to_cp437_ce(value1);
+            }
+            output.charactersUsed = 2;
+            break;
+        case 0xCF:
+            if(codepage == CODE_PAGE_737){
+                output.character = utf_to_cp737_cf(value1);
+            } else {
+                output.character = utf_to_cp437_cf(value1);
+            }
+            output.charactersUsed = 2;
+            break;
+        case 0xE2:
+            if(codepage == CODE_PAGE_737){
+                output.character = utf_to_cp737_e2(value1, value2);
+            } else {
+                output.character = utf_to_cp437_e2(value1, value2);
+            }
+            output.charactersUsed = 3;
+            break;
+        default:
+            output.character = UNKNOWN_CHAR_REPLACEMENT;
+            output.charactersUsed = 1;
+    }
+
+    return output;
+
+}
diff --git a/doschgpt-code/utf2cp.h b/doschgpt-code/utf2cp.h
@@ -0,0 +1,19 @@
+#define UNKNOWN_CHAR_REPLACEMENT 0xFE
+
+#define CODE_PAGE_737 737
+#define CODE_PAGE_437 437
+
+typedef struct
+{
+    //How many values was used during parsing
+    int charactersUsed;
+
+    //Page code of the character. Using int to avoid padding compilation problem
+    // If character cannot be matched, will return UNKNOWN_CHAR_REPLACEMENT
+    unsigned int character;
+
+} CONVERSION_OUTPUT;
+
+
+//Provide up to 3 bytes of the string to attempt to parse to code page
+CONVERSION_OUTPUT utf_to_cp(int codepage, int value0, int value1, int value2);
diff --git a/doschgpt-code/utfcp437.cpp b/doschgpt-code/utfcp437.cpp
@@ -1,48 +1,5 @@
 #include "utfcp437.h"
-
-CONVERSION_OUTPUT utf_to_cp437(int value0, int value1, int value2){
-    CONVERSION_OUTPUT output;
-
-    if(value0 <= 127){
-        output.character = value0;
-        output.charactersUsed = 1;
-        return output;
-    }
-
-    switch(value0){
-        case 0xC2:
-            output.character = utf_to_cp437_c2(value1);
-            output.charactersUsed = 2;
-            break;
-        case 0xC3:
-            output.character = utf_to_cp437_c3(value1);
-            output.charactersUsed = 2;
-            break;
-        case 0xC6:
-            output.character = utf_to_cp437_c6(value1);
-            output.charactersUsed = 2;
-            break;
-        case 0xCE:
-            output.character = utf_to_cp437_ce(value1);
-            output.charactersUsed = 2;
-            break;
-        case 0xCF:
-            output.character = utf_to_cp437_cf(value1);
-            output.charactersUsed = 2;
-            break;
-        case 0xE2:
-            output.character = utf_to_cp437_e2(value1, value2);
-            output.charactersUsed = 3;
-            break;
-        default:
-            output.character = UNKNOWN_CHAR_REPLACEMENT;
-            output.charactersUsed = 1;
-    }
-
-    return output;
-
-}
-
+#define UNKNOWN_CHAR_REPLACEMENT 0xFE
 
 unsigned char utf_to_cp437_c3(int value1){
     switch(value1){
@@ -208,7 +165,7 @@ unsigned char utf_to_cp437_e2(int value1, int value2){
         }
     }
 
-    return value1;
+    return UNKNOWN_CHAR_REPLACEMENT;
 }
 
 unsigned char utf_to_cp437_ce(int value1){

diff --git a/doschgpt-code/utfcp437.h b/doschgpt-code/utfcp437.h
@@ -1,21 +1,3 @@
-#define UNKNOWN_CHAR_REPLACEMENT 0xFE
-
-typedef struct
-{
-    //How many values was used during parsing
-    int charactersUsed;
-
-    //(Extended) ASCII code of the character. Using int to avoid padding compilation problem
-    // If character cannot be matched, will return UNKNOWN_CHAR_REPLACEMENT
-    unsigned int character;
-
-} CONVERSION_OUTPUT;
-
-
-//Provide up to 3 bytes of the string to attempt to merge to CP437 code page
-CONVERSION_OUTPUT utf_to_cp437(int value0, int value1, int value2);
-
-// Internally calls this to parse second characters starting with _cX
 unsigned char utf_to_cp437_c2(int value1);
 unsigned char utf_to_cp437_c3(int value1);
 unsigned char utf_to_cp437_c6(int value1);
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		{"id":"chatcmpl-73JoAf0rMfuEnKjayuVg0BBqWb18O","object":"chat.completion","created":1681025482,"model":"gpt-3.5-turbo-0301","usage":{"prompt_tokens":23,"completion_tokens":234,"total_tokens":257},"choices":[{"message":{"role":"assistant","content":"As an AI language model, I cannot display characters in a visual format as they are shown in a computer. However, I can list the characters included in the Greek Code Page 737:Α,Β,Γ,Δ,Ε,Ζ,Η,Θ,Ι,Κ,Λ,Μ,Ν,Ξ,Ο,Π,Ρ,Σ,Τ,Υ,Φ,Χ,Ψ,Ω,α,β,γ,δ,ε,ζ,η,θ,ι,κ,λ,μ,ν,ξ,ο,π,ρ,σ,ς,τ,υ,φ,χ,ψ,░,▒,▓,│,┤,╡,╢,╖,╕,╣,║,╗,╝,╜,╛,┐,└,┴,┬,├,─,┼,╞,╟,╚,╔,╩,╦,╠,═,╬,╧,╨,╤,╥,╙,╘,╒,╓,╫,╪,┘,┌,█,▄,▌,▐,▀,ω,ά,έ,ή,ϊ,ί,ό,ύ,ϋ,ώ,Ά,Έ,Ή,Ί,Ό,Ύ,Ώ,±,≥,≤,Ϊ,Ϋ,÷,≈,°,∙,·,√,ⁿ,²,■"},"finish_reason":"stop","index":0}]}
Original file line number	Diff line number	Diff line change
		@@ -1 +1 @@
		{"id":"chatcmpl-703fFOKPE7ffRr0ecA4V9snJ5lPEG","object":"chat.completion","created":1680248441,"model":"gpt-3.5-turbo-0301","usage":{"prompt_tokens":20,"completion_tokens":610,"total_tokens":630},"choices":[{"message":{"role":"assistant","content":"Here is a list of extended ASCII characters from 128 to 255:,,128: Ç,129: ü,130: é,131: â,132: ä,133: à,134: å,135: ç,136: ê,137: ë,138: è,139: ï,140: î,141: ì,142: Ä,143: Å,144: É,145: æ,146: Æ,147: ô,148: ö,149: ò,150: û,151: ù,152: ÿ,153: Ö,154: Ü,155: ¢,156: £,157: ¥,158: ₧,159: ƒ,160: á,161: í,162: ó,163: ú,164: ñ,165: Ñ,166: ª,167: º,168: ¿,169: ⌐,170: ¬,171: ½,172: ¼,173: ¡,174: «,175: »,176: ░,177: ▒,178: ▓,179: │,180: ┤,181: ╡,182: ╢,183: ╖,184: ╕,185: ╣,186: ║,187: ╗,188: ╝,189: ╜,190: ╛,191: ┐,192: └,193: ┴,194: ┬,195: ├,196: ─,197: ┼,198: ╞,199: ╟,200: ╚,201: ╔,202: ╩,203: ╦,204: ╠,205: ═,206: ╬,207: ╧,208: ╨,209: ╤,210: ╥,211: ╙,212: ╘,213: ╒,214: ╓,215: ╫,216: ╪,217: ┘,218: ┌,219: █,220: ▄,221: ▌,222: ▐,223: ▀,224: α,225: ß,226: Γ,227: π,228: Σ,229: σ,230: µ,231: τ,232: Φ,233: Θ,234: Ω,235: δ,236: ∞,237: φ,238: ε,239: ∩,240: ≡,241: ±,242: ≥,243: ≤,244: ⌠,245: ⌡,246: ÷,247: ≈,248: °,249: ∙,250: ·,251: √,252: ⁿ,253: ²,254: ■,255: "},"finish_reason":"stop","index":0}]}
		{"id":"chatcmpl-73JoAf0rMfuEnKjayuVg0BBqWb18O","object":"chat.completion","created":1681025482,"model":"gpt-3.5-turbo-0301","usage":{"prompt_tokens":23,"completion_tokens":234,"total_tokens":257},"choices":[{"message":{"role":"assistant","content":"As an AI language model, I cannot display characters in a visual format as they are shown in a computer. However, I can list the characters included in the Greek Code Page 737:Α,Β,Γ,Δ,Ε,Ζ,Η,Θ,Ι,Κ,Λ,Μ,Ν,Ξ,Ο,Π,Ρ,Σ,Τ,Υ,Φ,Χ,Ψ,Ω,α,β,γ,δ,ε,ζ,η,θ,ι,κ,λ,μ,ν,ξ,ο,π,ρ,σ,ς,τ,υ,φ,χ,ψ,░,▒,▓,│,┤,╡,╢,╖,╕,╣,║,╗,╝,╜,╛,┐,└,┴,┬,├,─,┼,╞,╟,╚,╔,╩,╦,╠,═,╬,╧,╨,╤,╥,╙,╘,╒,╓,╫,╪,┘,┌,█,▄,▌,▐,▀,ω,ά,έ,ή,ϊ,ί,ό,ύ,ϋ,ώ,Ά,Έ,Ή,Ί,Ό,Ύ,Ώ,±,≥,≤,Ϊ,Ϋ,÷,≈,°,∙,·,√,ⁿ,²,■"},"finish_reason":"stop","index":0}]}