Unicode lib, use terminal in readline, tests

CMakeLists.txt

@@ -20,6 +20,7 @@ set(SRC
     "src/Test.c"
     "src/Thread.c"
     "src/ToString.c"
+    "src/Unicode.c"
     "src/Utility.c"
     "src/Vector.c"
     "src/View.c"
@@ -43,6 +44,7 @@ set(SRC_TESTS
     "test/modules/SpinLockTests.c"
     "test/modules/TerminalTests.c"
     "test/modules/TestTests.c"
+    "test/modules/UnicodeTests.c"
     "test/modules/UtilityTests.c"
     "test/modules/VectorTests.c"
     "test/modules/ViewTests.c"
@@ -110,10 +112,12 @@ target_sources(core PUBLIC
         ./include/core/Random.h
         ./include/core/ReadLine.h
         ./include/core/SpinLock.h
+        ./include/core/Terminal.h
         ./include/core/Test.h
         ./include/core/Thread.h
         ./include/core/ToString.h
         ./include/core/Types.h
+        ./include/core/Unicode.h
         ./include/core/Utility.h
         ./include/core/Vector.h
         ./include/core/View.h

include/core/Terminal.h

@@ -63,12 +63,13 @@
 #define TERMINAL_KEY_ARROW_RIGHT 0x3'0000'0000lu
 #define TERMINAL_KEY_ARROW_UP 0x4'0000'0000lu
 #define TERMINAL_KEY_ARROW_DOWN 0x5'0000'0000lu
-#define TERMINAL_KEY_ARROW_DELETE 0x6'0000'0000lu
+#define TERMINAL_KEY_DELETE 0x6'0000'0000lu
 
 void enterAlternativeTerminal();
 void leaveAlternativeTerminal();
 IntVector2 getTerminalSize();
 void clearTerminal();
+void clearTerminalLine();
 
 void hideCursor();
 void showCursor();

include/core/Unicode.h

@@ -0,0 +1,16 @@
+#ifndef CORE_UNICODE_H
+#define CORE_UNICODE_H
+
+#include "core/Types.h"
+
+typedef struct {
+    u8 data[4];
+    u32 length;
+} UTF8;
+
+UTF8 convertUnicodeToUTF8(u32 c);
+u32 convertUTF8toUnicode(UTF8 c);
+bool isUTF8Remainder(u8 c);
+u32 getUTF8Length(u8 c);
+
+#endif

src/ReadLine.c

@@ -1,18 +1,16 @@
 #include "core/ReadLine.h"
 
 #include <ctype.h>
-#include <errno.h>
 #include <stdatomic.h>
 #include <stdint.h>
 #include <stdio.h>
 #include <string.h>
-#include <termios.h>
-#include <unistd.h>
 
 #include "ErrorSimulator.h"
 #include "core/Logger.h"
 #include "core/Queue.h"
 #include "core/Thread.h"
+#include "core/Unicode.h"
 
 static constexpr size_t HISTORY_LENGTH = 10;
 static constexpr size_t CONSOLE_BUFFER_SIZE = 256;
@@ -28,7 +26,6 @@ QUEUE_SOURCE(ConsoleLine, CL)
 static atomic_bool running = true;
 static thrd_t readThread = {0};
 
-static struct termios original;
 static QueueCL buffer = {0};
 static ConsoleLine currentBuffer = {0};
 static size_t move = 0;
@@ -40,35 +37,20 @@ static size_t historyOffset = 0;
 static size_t historyIndex = 0;
 static size_t historyLength = 0;
 
-static void getAttributes(struct termios* t) {
-    if(tcgetattr(STDIN_FILENO, t)) {
-        LOG_WARNING("cannot get terminal attributes");
-    }
-}
-
-static void restoreAttributes() {
-    if(tcsetattr(STDIN_FILENO, TCSAFLUSH, &original)) {
-        LOG_WARNING("cannot restore terminal attributes: %s", strerror(errno));
-    }
-}
-
-static char readChar() {
-    char c = 0;
-    ssize_t bytes = read(STDIN_FILENO, &c, 1);
-    return bytes <= 0 ? '\0' : c;
-}
-
-static void addChar(char c) {
-    if(currentBuffer.length >= CONSOLE_BUFFER_SIZE - 1) {
-        return;
-    }
-    for(size_t i = 0; i < move; i++) {
-        currentBuffer.data[currentBuffer.length - i] =
-            currentBuffer.data[currentBuffer.length - i - 1];
+static void addChar(u64 c) {
+    UTF8 u = convertUnicodeToUTF8((u32)c);
+    for(u32 k = 0; k < u.length; k++) {
+        if(currentBuffer.length >= CONSOLE_BUFFER_SIZE - 1) {
+            return;
+        }
+        for(size_t i = 0; i < move; i++) {
+            currentBuffer.data[currentBuffer.length - i] =
+                currentBuffer.data[currentBuffer.length - i - 1];
+        }
+        currentBuffer.length++;
+        currentBuffer.data[currentBuffer.length - move - 1] = (char)u.data[k];
+        currentBuffer.data[currentBuffer.length] = '\0';
     }
-    currentBuffer.length++;
-    currentBuffer.data[currentBuffer.length - move - 1] = c;
-    currentBuffer.data[currentBuffer.length] = '\0';
 }
 
 static void print(const char* s) {
@@ -79,10 +61,10 @@ static void refreshLine(const char* prefix) {
     print(prefix);
     print(currentBuffer.data);
     if(cursorMove > 0) {
-        printf("\33[%zuD", cursorMove);
+        moveCursorLeft((int)cursorMove);
     }
     fflush(stdout);
-    print("\33[2K\r");
+    clearTerminalLine();
 }
 
 static bool clear() {
@@ -122,16 +104,12 @@ static void addLine() {
     clear();
 }
 
-static char isUTF8Part(char c) {
-    return ((uint8_t)c & 0xC0) == 0x80;
-}
-
 static bool removeChar() {
     size_t pos = currentBuffer.length - move;
     if(pos > 0) {
         size_t l = 1;
         while(pos - l >= 0) {
-            if(!isUTF8Part(currentBuffer.data[pos - l])) {
+            if(!isUTF8Remainder((u8)currentBuffer.data[pos - l])) {
                 break;
             }
             l++;
@@ -144,7 +122,7 @@ static bool removeChar() {
     return false;
 }
 
-static bool handleControlKey(char c) {
+static bool handleControlKey(u64 c) {
     switch(c) {
         case 3: return clear();
         case 10:
@@ -184,7 +162,7 @@ static char getMoved() {
 static void handleLeftArrow() {
     if(move < currentBuffer.length) {
         move++;
-        while(move < currentBuffer.length && isUTF8Part(getMoved())) {
+        while(move < currentBuffer.length && isUTF8Remainder((u8)getMoved())) {
             move++;
         }
         cursorMove++;
@@ -194,38 +172,31 @@ static void handleLeftArrow() {
 static void handleRightArrow() {
     if(move > 0) {
         move--;
-        while(move > 0 && isUTF8Part(getMoved())) {
+        while(move > 0 && isUTF8Remainder((u8)getMoved())) {
             move--;
         }
         cursorMove--;
     }
 }
 
-static void readEscapeSequence() {
-    if(readChar() != '[') {
-        return;
-    }
-    switch(readChar()) {
-        case 'A': handleUpArrow(); break;
-        case 'B': handleDownArrow(); break;
-        case 'C': handleRightArrow(); break;
-        case 'D': handleLeftArrow(); break;
-        case '3':
-            if(readChar() == '~' && move > 0) {
-                handleRightArrow();
-                removeChar();
-            }
-            break;
-    }
-}
-
 static void handleChars() {
     while(true) {
-        char c = readChar();
+        u64 c = getRawChar();
         if(c == 0) {
             break;
-        } else if(c == 27) {
-            readEscapeSequence();
+        } else if(c == TERMINAL_KEY_ARROW_UP) {
+            handleUpArrow();
+        } else if(c == TERMINAL_KEY_ARROW_DOWN) {
+            handleDownArrow();
+        } else if(c == TERMINAL_KEY_ARROW_RIGHT) {
+            handleRightArrow();
+        } else if(c == TERMINAL_KEY_ARROW_LEFT) {
+            handleLeftArrow();
+        } else if(c == TERMINAL_KEY_DELETE) {
+            if(move > 0) {
+                handleRightArrow();
+                removeChar();
+            }
         } else if(iscntrl(c)) {
             if(handleControlKey(c)) {
                 addLine();
@@ -249,18 +220,9 @@ static int loop(void* data) {
 }
 
 bool startReadLine(void) {
-    // https://viewsourcecode.org/snaptoken/kilo/02.enteringRawMode.html
-    getAttributes(&original);
-    struct termios raw = original;
-    raw.c_iflag &= ~(tcflag_t)(ICRNL | IXON);
-    raw.c_lflag &= ~(tcflag_t)(ECHO | ICANON | IEXTEN | ISIG);
-    raw.c_cc[VMIN] = 0;
-    raw.c_cc[VTIME] = 0;
-
-    if(tcsetattr(STDIN_FILENO, TCSANOW, &raw)) {
+    if(enterRawTerminal()) {
         LOG_WARNING("cannot set terminal attributes");
     }
-
     initQueueCL(&buffer, 10);
     atomic_store(&running, true);
     if(MUTEX_INIT_FAIL || mtx_init(&bufferMutex, mtx_plain) != thrd_success) {
@@ -292,7 +254,9 @@ bool readLine(char* buffer_, size_t n) {
 void stopReadLine() {
     atomic_store(&running, false);
     joinThreadSafe(&readThread);
-    restoreAttributes();
+    if(leaveRawTerminal()) {
+        LOG_WARNING("cannot restore terminal attributes");
+    }
     destroyQueueCL(&buffer);
     mtx_destroy(&bufferMutex);
 }

src/Terminal.c

@@ -5,6 +5,8 @@
 #include <termios.h>
 #include <unistd.h>
 
+#include "core/Unicode.h"
+
 #define ESC "\33["
 #define esc(s) fputs(ESC s, stdout)
 
@@ -30,6 +32,10 @@ IntVector2 getTerminalSize() {
     return IV(w.ws_col, w.ws_row);
 }
 
+void clearTerminalLine() {
+    esc("2K\r");
+}
+
 void hideCursor() {
     esc("?25l");
 }
@@ -91,26 +97,26 @@ static u64 readEscapeSequence() {
         case 'D': return TERMINAL_KEY_ARROW_LEFT;
         case '3':
             if(readChar() == '~') {
-                return TERMINAL_KEY_ARROW_DELETE;
+                return TERMINAL_KEY_DELETE;
             }
             break;
     }
     return TERMINAL_KEY_UNKNOWN;
 }
 
-static u64 readUTF8(u64 c) {
-    if((c & 0b1110'0000) == 0b1100'0000) {
-        return ((c & 0b0001'1111) << 6) | (readChar() & 0b0011'1111);
+static u64 readUTF8() {
+    UTF8 u = {0};
+    u.data[0] = readChar();
+    u.length = getUTF8Length(u.data[0]);
+    for(u32 i = 1; i < u.length; i++) {
+        u.data[i] = readChar();
     }
-    // TODO: 3/4 byte UTF8 characters
-    return c;
+    return convertUTF8toUnicode(u);
 }
 
 u64 getRawChar() {
-    u8 c = readChar();
-    if(c > 127) {
-        return readUTF8(c);
-    } else if(c == 27) {
+    u64 c = readUTF8();
+    if(c == 27) {
         return readEscapeSequence();
     }
     return c;

src/Unicode.c

@@ -0,0 +1,54 @@
+#include "core/Unicode.h"
+
+UTF8 convertUnicodeToUTF8(u32 c) {
+    UTF8 u = {0};
+    if(c >= 0x10000) {
+        u.length = 4;
+        u.data[0] = 0b1111'0000 | ((c >> 18) & 0b0000'0111);
+        u.data[1] = 0b1000'0000 | ((c >> 12) & 0b0011'1111);
+        u.data[2] = 0b1000'0000 | ((c >> 6) & 0b0011'1111);
+        u.data[3] = 0b1000'0000 | (c & 0b0011'1111);
+    } else if(c >= 0x800) {
+        u.length = 3;
+        u.data[0] = 0b1110'0000 | ((c >> 12) & 0b0000'1111);
+        u.data[1] = 0b1000'0000 | ((c >> 6) & 0b0011'1111);
+        u.data[2] = 0b1000'0000 | (c & 0b0011'1111);
+    } else if(c >= 0x80) {
+        u.length = 2;
+        u.data[0] = 0b1100'0000 | ((c >> 6) & 0b0001'1111);
+        u.data[1] = 0b1000'0000 | (c & 0b0011'1111);
+    } else {
+        u.length = 1;
+        u.data[0] = c & 0b0111'1111;
+    }
+    return u;
+}
+
+u32 convertUTF8toUnicode(UTF8 c) {
+    if(c.length == 4) {
+        return ((c.data[0] & 0b0000'0111u) << 18) |
+               ((c.data[1] & 0b0011'1111u) << 12) |
+               ((c.data[2] & 0b0011'1111u) << 6) | (c.data[3] & 0b0011'1111u);
+    } else if(c.length == 3) {
+        return ((c.data[0] & 0b0000'1111u) << 12) |
+               ((c.data[1] & 0b0011'1111u) << 6) | (c.data[2] & 0b0011'1111u);
+    } else if(c.length == 2) {
+        return ((c.data[0] & 0b0001'1111u) << 6) | (c.data[1] & 0b0011'1111u);
+    }
+    return c.data[0];
+}
+
+bool isUTF8Remainder(u8 c) {
+    return (c & 0b1100'0000) == 0b1000'0000;
+}
+
+u32 getUTF8Length(u8 c) {
+    if((c & 0b1111'1000) == 0b1111'0000) {
+        return 4;
+    } else if((c & 0b1111'0000) == 0b1110'0000) {
+        return 3;
+    } else if((c & 0b1110'0000) == 0b1100'0000) {
+        return 2;
+    }
+    return 1;
+}

test/Main.c

@@ -8,22 +8,26 @@
 #include "core/Test.h"
 #include "core/Utility.h"
 
+static void finalize() {
+    finalizeTests();
+    printMemoryReport();
+}
+
 static void onExit(int code, void* data) {
     unsigned int i = *(unsigned int*)(data);
     LOG_WARNING("Hello from exit %d: %u", code, i);
-    finalizeTests();
-    printMemoryReport();
+    finalize();
 }
 
 int main(int argAmount, const char** args) {
     if(argAmount >= 2 && strcmp(args[1], "help") == 0) {
-        // puts("alloc");
-        // puts("realloc");
-        // puts("pre_canary");
-        // puts("post_canary");
-        // puts("test;ignore");
-        // puts("light;readLineTest");
+        puts("alloc");
+        puts("realloc");
+        puts("pre_canary");
+        puts("post_canary");
+        puts("test;ignore");
         puts("terminal");
+        puts("light;readLineTest");
         return 0;
     }
     setlocale(LC_ALL, "en_US.utf8");
@@ -43,7 +47,11 @@ int main(int argAmount, const char** args) {
             testTest();
             return 0;
         } else if(strcmp(args[i], "terminal") == 0) {
-            testTerminal();
+            testTerminal(true);
+            finalize();
+            return 0;
+        } else if(strcmp(args[i], "iterminal") == 0) {
+            testInteractiveTerminal();
             return 0;
         }
     }
@@ -61,9 +69,13 @@ int main(int argAmount, const char** args) {
     testQueue();
     testRandom(light);
     logLevel = LOG_NONE;
-    testReadLine();
+    if(light) {
+        testReadLine();
+    }
     logLevel = LOG_DEBUG;
     testSpinLock();
+    testTerminal(!light);
+    testUnicode();
     testUtility(light);
     testVector();
     testView();

test/Tests.h

@@ -11,6 +11,7 @@ void testBuffer(bool light);
 void testComponents(void);
 void testFrustum(void);
 void testHashMap(bool light);
+void testInteractiveTerminal(void);
 void testList(bool light);
 void testMatrix(void);
 void testPlane(void);
@@ -21,8 +22,9 @@ void testQueue(void);
 void testRandom(bool light);
 void testReadLine(void);
 void testSpinLock(void);
-void testTerminal(void);
+void testTerminal(bool tty);
 void testTest(void);
+void testUnicode(void);
 void testUtility(bool light);
 void testVector(void);
 void testView(void);

test/modules/TerminalTests.c

@@ -8,7 +8,36 @@
 #define KEY_CASE(key)           \
     case key: puts(#key); break
 
-void testTerminal(void) {
+void testTerminal(bool tty) {
+    IntVector2 size = getTerminalSize();
+    if(tty) {
+        TEST_TRUE(size.x > 0);
+        TEST_TRUE(size.y > 0);
+        TEST_FALSE(enterRawTerminal());
+        TEST_FALSE(leaveRawTerminal());
+    } else {
+        TEST_INT(0, size.x);
+        TEST_INT(0, size.y);
+    }
+
+    enterAlternativeTerminal();
+    clearTerminal();
+    resetCursor();
+    LOG_ERROR("Not visible!");
+    leaveAlternativeTerminal();
+
+    LOG_ERROR("Not visible!");
+    moveCursorUp(2);
+    moveCursorDown(1);
+    moveCursorLeft(3);
+    moveCursorRight(3);
+    clearTerminalLine();
+
+    hideCursor();
+    showCursor();
+}
+
+void testInteractiveTerminal(void) {
     enterAlternativeTerminal();
 
     resetCursor();
@@ -37,7 +66,7 @@ void testTerminal(void) {
             KEY_CASE(TERMINAL_KEY_ARROW_RIGHT);
             KEY_CASE(TERMINAL_KEY_ARROW_UP);
             KEY_CASE(TERMINAL_KEY_ARROW_DOWN);
-            KEY_CASE(TERMINAL_KEY_ARROW_DELETE);
+            KEY_CASE(TERMINAL_KEY_DELETE);
             default: printf("%lu\n", c); break;
         }
     }

test/modules/UnicodeTests.c

@@ -0,0 +1,14 @@
+#include "../Tests.h"
+#include "core/Unicode.h"
+
+void testUnicode(void) {
+    for(u32 c = 0; c < 0x10FFFF; c += 50) {
+        UTF8 u = convertUnicodeToUTF8(c);
+        for(u32 k = 1; k < u.length; k++) {
+            TEST_TRUE(isUTF8Remainder(u.data[k]));
+        }
+        TEST_TRUE(u.length >= 1);
+        TEST_U32(u.length, getUTF8Length(u.data[0]));
+        TEST_U32(c, convertUTF8toUnicode(u));
+    }
+}

test/modules/UtilityTests.c

@@ -73,6 +73,14 @@ static void testSleep(i64 nanos) {
     TEST_TRUE(time >= nanos && time <= (nanos * 13) / 10);
 }
 
+static void testSleepMillis(i64 millis) {
+    i64 time = -getNanos();
+    TEST_FALSE(sleepMillis(millis));
+    time += getNanos();
+    i64 nanos = millis * 1'000'000;
+    TEST_TRUE(time >= nanos && time <= (nanos * 13) / 10);
+}
+
 typedef struct {
     int i;
     i64 d;
@@ -134,6 +142,8 @@ void testUtility(bool light) {
     testDegreeToRadian();
     testSleep(light ? 5'000'000 : 50'000'000);
     testSleep(light ? 50'000'000 : 1'300'000'000);
+    testSleepMillis(light ? 5 : 50);
+    testSleepMillis(light ? 50 : 1300);
     testSwap();
     testFail();
     testSort();