ECE 408 sem project

utils/tokenizer.h

@@ -0,0 +1,108 @@
+//-----------------------------------------------------------------------------------------------
+// Copyright (c) 2024 Andrej Karpathy
+// Licensed under the MIT License. See the LICENSE file for details.
+//
+// Modifications Copyright (c) 2025 Hanwen Liu, Hrishi Shah, Kelin Zeng, Charles Pei, and Vijay Daita, ALL RIGHTS RESERVED.
+//-----------------------------------------------------------------------------------------------
+
+/*
+Defines the GPT-2 Tokenizer.
+Only supports decoding, i.e.: tokens (integers) -> strings
+*/
+#ifndef UTILS_TOKENIZER_H
+#define UTILS_TOKENIZER_H
+
+#include <stdint.h>
+#include <ctype.h>
+#include <assert.h>
+#include "utils.h"
+
+// ----------------------------------------------------------------------------
+
+typedef struct {
+    uint32_t vocab_size;
+    char **token_table;
+    int init_ok;
+    int eot_token; // <|endoftext|> token id
+} Tokenizer;
+
+void safe_printf(const char *piece) {
+    // the tokens are raw bytes, and we we only want to print the printable ones
+    // many bytes can be various control codes, backspace, etc.
+    if (piece == NULL) { return; }
+    if (piece[0] == '\0') { return; }
+    // handle individual byte tokens
+    // every token is asserted to be at least one byte so doing piece[1] is ok
+    if (piece[1] == '\0') {
+        unsigned char byte_val = piece[0];
+        if (!(isprint(byte_val) || isspace(byte_val))) {
+            return; // weird byte, don't print it
+        }
+    }
+    printf("%s", piece);
+}
+
+void tokenizer_init(Tokenizer *tokenizer, const char *filename) {
+    FILE *file = fopen(filename, "rb");
+    if (file == NULL) {
+        printf("---\n");
+        printf("Tokenizer - WARNING: Failed to open the tokenizer file %s\n", filename);
+        printf("---\n");
+        tokenizer->init_ok = 0;
+        return;
+    }
+    // read in the header
+    uint32_t header[256];
+    freadCheck(header, sizeof(uint32_t), 256, file);
+    assert(header[0] == 20240328);
+    int version = header[1];
+    tokenizer->vocab_size = header[2];
+    if (version == 1) {
+        // version 1 didn't include the EOT token id
+        // so we assume it is 50256, the EOT in GPT-2
+        assert(tokenizer->vocab_size == 50257); // let's be defensive here
+        tokenizer->eot_token = 50256;
+    } else if (version == 2) {
+        tokenizer->eot_token = header[3];
+    } else {
+        fprintf(stderr, "Tokenizer model file %s has bad version: %d\n", filename, version);
+        exit(EXIT_FAILURE);
+    }
+    // read in all the tokens
+    unsigned char length;
+    tokenizer->token_table = (char **)mallocCheck(tokenizer->vocab_size * sizeof(char *));
+    for (uint32_t i = 0; i < tokenizer->vocab_size; i++) {
+        freadCheck(&length, sizeof(unsigned char), 1, file);
+        assert(length > 0); // every token should be at least one character
+        char *token_bytes = (char *)mallocCheck(length + 1);
+        freadCheck(token_bytes, sizeof(char), length, file);
+        token_bytes[length] = '\0';  // Add null terminator for printing
+        tokenizer->token_table[i] = token_bytes;
+    }
+    // cleanups
+    fcloseCheck(file);
+    tokenizer->init_ok = 1;
+}
+
+const char *tokenizer_decode(Tokenizer *tokenizer, uint32_t token_id) {
+    if (tokenizer->init_ok == 0) {
+        return NULL;
+    }
+    if (token_id < tokenizer->vocab_size) {
+        return tokenizer->token_table[token_id];
+    } else {
+        printf("Tokenizer - Invalid token id %u!\n", token_id);
+        return NULL;
+    }
+}
+
+void tokenizer_free(Tokenizer *tokenizer) {
+    if (tokenizer->init_ok) {
+        for (uint32_t i = 0; i < tokenizer->vocab_size; i++) {
+            free(tokenizer->token_table[i]);
+        }
+        free(tokenizer->token_table);
+    }
+}
+
+#endif  // UTILS_TOKENIZER_H