token - Gstar7CodeMan/testings GitHub Wiki

Sure! Here is a step-by-step guide to building a .NET Core application that meets your requirements. I'll break down the response into different sections for clarity.

1. Create a new ASP.NET Core Web Application:

   dotnet new webapp -n TokenizationCostEstimator
   cd TokenizationCostEstimator

2. Add necessary packages:

   • Add packages for file processing, like EPPlus for Excel files and RestSharp for API calls.

   dotnet add package EPPlus
   dotnet add package RestSharp

1. Create a folder Models and add FileDetails.cs:

   public class FileDetails
   {
       public int NumberOfAlphabets { get; set; }
       public int NumberOfNumbers { get; set; }
       public int NumberOfWhitespaces { get; set; }
       public int NumberOfSpecialCharacters { get; set; }
       public int TotalWordCount { get; set; }
       public double AverageWordLength { get; set; }
       public int SentenceCount { get; set; }
       public int ParagraphCount { get; set; }
   }

2. Add TokenizationResult.cs:

   public class TokenizationResult
   {
       public int TokenSize { get; set; }
       public Dictionary<string, double> CostsByModel { get; set; }
   }

1. Create a folder Services and add FileProcessingService.cs:

   using OfficeOpenXml;
   using System.Text.RegularExpressions;
   
   public class FileProcessingService
   {
       public FileDetails AnalyzeTextFile(string content)
       {
           var details = new FileDetails();
           details.NumberOfAlphabets = Regex.Matches(content, @"[a-zA-Z]").Count;
           details.NumberOfNumbers = Regex.Matches(content, @"\d").Count;
           details.NumberOfWhitespaces = Regex.Matches(content, @"\s").Count;
           details.NumberOfSpecialCharacters = Regex.Matches(content, @"\W").Count;
           details.TotalWordCount = content.Split(' ', StringSplitOptions.RemoveEmptyEntries).Length;
           details.AverageWordLength = content.Split(' ', StringSplitOptions.RemoveEmptyEntries).Average(w => w.Length);
           details.SentenceCount = Regex.Matches(content, @"[.!?]").Count;
           details.ParagraphCount = content.Split(new[] { "\r\n\r\n" }, StringSplitOptions.None).Length;
           return details;
       }
   
       public FileDetails AnalyzeExcelFile(byte[] fileBytes)
       {
           using var package = new ExcelPackage(new MemoryStream(fileBytes));
           var details = new FileDetails();
           foreach (var sheet in package.Workbook.Worksheets)
           {
               var content = sheet.Cells[sheet.Dimension.Address].Text;
               var sheetDetails = AnalyzeTextFile(content);
               // Aggregate results for all sheets
               details.NumberOfAlphabets += sheetDetails.NumberOfAlphabets;
               details.NumberOfNumbers += sheetDetails.NumberOfNumbers;
               details.NumberOfWhitespaces += sheetDetails.NumberOfWhitespaces;
               details.NumberOfSpecialCharacters += sheetDetails.NumberOfSpecialCharacters;
               details.TotalWordCount += sheetDetails.TotalWordCount;
               details.AverageWordLength += sheetDetails.AverageWordLength;
               details.SentenceCount += sheetDetails.SentenceCount;
               details.ParagraphCount += sheetDetails.ParagraphCount;
           }
           details.AverageWordLength /= package.Workbook.Worksheets.Count;
           return details;
       }
   }

2. Add OpenAiService.cs:

   using RestSharp;
   
   public class OpenAiService
   {
       private readonly string _apiUrl = "https://platform.openai.com/tokenizer";
   
       public async Task<int> GetTokenSizeAsync(string content)
       {
           var client = new RestClient(_apiUrl);
           var request = new RestRequest(Method.POST);
           request.AddParameter("text/plain", content, ParameterType.RequestBody);
           var response = await client.ExecuteAsync(request);
           return int.Parse(response.Content);
       }
   
       public Dictionary<string, double> CalculateCosts(int tokenSize)
       {
           var costs = new Dictionary<string, double>
           {
               { "GPT-3", tokenSize * 0.0004 },
               { "GPT-3.5", tokenSize * 0.0006 },
               { "GPT-4", tokenSize * 0.001 }
           };
           return costs;
       }
   }

1. Create a folder Controllers and add FileUploadController.cs:

   using Microsoft.AspNetCore.Http;
   using Microsoft.AspNetCore.Mvc;
   
   [ApiController]
   [Route("[controller]")]
   public class FileUploadController : ControllerBase
   {
       private readonly FileProcessingService _fileProcessingService;
       private readonly OpenAiService _openAiService;
   
       public FileUploadController(FileProcessingService fileProcessingService, OpenAiService openAiService)
       {
           _fileProcessingService = fileProcessingService;
           _openAiService = openAiService;
       }
   
       [HttpPost]
       [Route("upload")]
       public async Task<IActionResult> Upload(IFormFile file)
       {
           if (file == null || file.Length == 0)
               return BadRequest("No file uploaded");
   
           var extension = Path.GetExtension(file.FileName).ToLower();
           string content;
           FileDetails details;
   
           using (var stream = new MemoryStream())
           {
               await file.CopyToAsync(stream);
               stream.Position = 0;
   
               if (extension == ".txt")
               {
                   using (var reader = new StreamReader(stream))
                   {
                       content = await reader.ReadToEndAsync();
                   }
                   details = _fileProcessingService.AnalyzeTextFile(content);
               }
               else if (extension == ".xlsx")
               {
                   var fileBytes = stream.ToArray();
                   content = ""; // Excel files will be processed sheet by sheet
                   details = _fileProcessingService.AnalyzeExcelFile(fileBytes);
               }
               else
               {
                   return BadRequest("Unsupported file type");
               }
           }
   
           var tokenSize = await _openAiService.GetTokenSizeAsync(content);
           var costs = _openAiService.CalculateCosts(tokenSize);
   
           var result = new TokenizationResult
           {
               TokenSize = tokenSize,
               CostsByModel = costs
           };
   
           return Ok(new { FileDetails = details, TokenizationResult = result });
       }
   }

1. Add services to Startup.cs or Program.cs (depending on your .NET Core version):

   public void ConfigureServices(IServiceCollection services)
   {
       services.AddControllers();
       services.AddScoped<FileProcessingService>();
       services.AddScoped<OpenAiService>();
       // Other service configurations
   }
   
   public void Configure(IApplicationBuilder app, IWebHostEnvironment env)
   {
       if (env.IsDevelopment())
       {
           app.UseDeveloperExceptionPage();
       }
   
       app.UseRouting();
       app.UseEndpoints(endpoints =>
       {
           endpoints.MapControllers();
       });
   }

1. In wwwroot, create an HTML file index.html:

   <!DOCTYPE html>
   <html lang="en">
   <head>
       <meta charset="UTF-8">
       <meta name="viewport" content="width=device-width, initial-scale=1.0">
       <title>Tokenization Cost Estimator</title>
       <link rel="stylesheet" href="https://stackpath.bootstrapcdn.com/bootstrap/4.5.2/css/bootstrap.min.css">
   </head>
   <body>
       <div class="container">
           <h1 class="mt-5">Tokenization Cost Estimator for OpenAI Models</h1>
           <form id="uploadForm">
               <div class="form-group">
                   <label for="fileInput">Upload your file:</label>
                   <input type="file" class="form-control-file" id="fileInput">
               </div>
               <button type="submit" class="btn btn-primary">Upload</button>
           </form>
           <div id="results" class="mt-5"></div>
       </div>
       <script>
           document.getElementById('uploadForm').addEventListener('submit', function(event) {
               event.preventDefault();
               var formData = new FormData();
               var fileInput = document.getElementById('fileInput');
               formData.append('file', fileInput.files[0]);
   
               fetch('/FileUpload/upload', {
                   method: 'POST',
                   body: formData
               }).then(response => response.json())
                 .then(data => {
                     var resultsDiv = document.getElementById('results');
                     resultsDiv.innerHTML = `<pre>${JSON.stringify(data, null, 2)}</pre>`;
                 }).catch(error => {
                     console.error('Error:', error);
                 });
           });
       </script>
   </body>
   </html>

1. Build and run the application:

   dotnet run

Navigate to http://localhost:5000 (or the specified port) to view and use the Tokenization Cost Estimator.