Tuesday, September 3, 2024

AWK - Fast Urlencoder - Handles Multibytes

#!/bin/bash
function urlencode() { 
LC_ALL=C awk 'BEGIN {
  for (i = 0; i <= 255; i++)
    hex[sprintf("%c", i)] = sprintf("%%%02X", i)
  safe = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_.:/!~*@$&[]+=,;?"
  split(safe, safechars, "")
  for (i in safechars)
    hex[safechars[i]] = safechars[i]
}
{
  encoded = ""
  for (i = 1; i <= length($0); i++)
    encoded = encoded hex[substr($0, i, 1)]
  print encoded
}
'
}
 This little bad boy rivals the speed of my C implementation below! 
 Slower by less than 10%.
 
 [Btw: this example is wrapped in a Bash function for ease of use]

Wednesday, August 28, 2024

Small Fast URL Encoder - vx_urlenc

// SPDX-License-Identifier: Apache-2.0
/***********************************************************
 **********************************************************
 
 * Fast URL Encoder - Closely conforms to RFC 3986
 *
 * Takes piped in uri's similar to jq but is quicker and conforms 
 * more to RFC 3986 [diverges w/most browsers on #'s]. 
 *
 * Handles multibyte unicode characters 
 *  
 * Copyright (C) 2024 Victrixsoft
 *
 * Issues: If you find any issues please open a ticket on github!
 *         https://github.com/victrixsoft/vx_urlenc/issues 
 * 
 * Author: Adam Danischewski <my first nm(dot)my last nm@gmail.com>
 * 
 **********************************************************
 ***********************************************************/

#include <stdio.h>
#include <stdlib.h>
#include <string.h>

#ifdef _WIN32
#include <io.h>
#define access _access
#else
#include <unistd.h>
#endif

#define ASCII_SIZE 128

char *url_encode(const char *);
void init_lookup_table(void);
int fileno(FILE *);
void __attribute__((constructor)) init(void);

char lookup_table[ASCII_SIZE] = {0};

void init_lookup_table(void) {
  for (int i = 'a'; i <= 'z'; i++)
    lookup_table[i] = 1;
  for (int i = 'A'; i <= 'Z'; i++)
    lookup_table[i] = 1;
  for (int i = '0'; i <= '9'; i++)
    lookup_table[i] = 1;
  const char *safe = "-_.:/!~*@$&[]+=,;?";
  while (*safe)
    lookup_table[(unsigned char)*safe++] = 1;
}

void __attribute__((constructor)) init() { init_lookup_table(); }

char *url_encode(const char *str) {
  if (str == NULL)
    return NULL;

  char *encoded =
      malloc(strlen(str) * 4 * 3 + 1); // Worst case: every char 4 byte mb
  if (encoded == NULL)
    return NULL;

  char *penc = encoded;
  size_t output_idx = 0;
  while (*str && *str != '\n') { // Stop at newline
    if (lookup_table[(unsigned char)*str]) {
      *penc++ = *str;
      output_idx++;
    } else {
      sprintf(penc, "%%%02X", (unsigned char)*str);
      penc += 3;
      output_idx += 3;
    }
    str++;
  }
  *penc = '\0';
  char *resized_enc = realloc(encoded, output_idx + 1);
  if (resized_enc == NULL) {
    return encoded;
  } else {
    return resized_enc;
  }
}

int main(int argc, char *argv[]) {
  (void)argc;
  (void)argv;

  if (isatty(fileno(stdin))) {
    perror("It's not a pipe\n");
    return 1;
  }

  char strbuf[65536];
  while (fgets(strbuf, sizeof strbuf, stdin) != NULL) {
    char *encoded = url_encode(strbuf);
    if (encoded) {
      printf("%s\n", encoded);
      free(encoded);
    } else {
      fprintf(stderr, "Error encoding line\n");
    }
  }

  return 0;
}
 Compile: gcc -o vx_urlenc vx_urlenc.c
   Usage: echo 'http://www.gaggle.com/🐀🐁🐂🐃🐄🐅🐆🐇🐈🐉🐊🐋🐌🐍!!![]*?#@.mp4' | ./vx_urlenc
 
https://github.com/victrixsoft/vx_urlenc

Wednesday, August 7, 2024

Unicode Codepoint to Hex - The Bash Way

function cp2hex() { 
 local codepoint="${1#U}"
 local -i bytes=0
 codepoint=${codepoint^^}
 local binary=$(echo "ibase=16;obase=2;${codepoint}"|bc)
 local -a binary_arr
  if ((0x${codepoint} <= 0x007F)); then
  bytes=1
  binary=$(printf "%08d" $binary)
  read -d=' ' -a binary_arr < <(sed 's/./ &/g' <<< "$binary")
  elif ((0x${codepoint} <= 0x07FF)); then
   bytes=2
   binary=$(printf "%011d" $binary)
   read -d=' ' -a binary_arr < <(sed 's/./ &/g' <<< "$binary")
   binary_arr[0]="110${binary_arr[0]}"
   binary_arr[5]="10${binary_arr[5]}"
  elif ((0x${codepoint} <= 0xFFFF)); then
   bytes=3
   binary=$(printf "%016d" $binary)
   read -d=' ' -a binary_arr < <(sed 's/./ &/g' <<< "$binary")
   binary_arr[0]="1110${binary_arr[0]}"
   binary_arr[4]="10${binary_arr[4]}"
   binary_arr[10]="10${binary_arr[10]}"
  elif ((0x${codepoint} <= 0x10FFFF)); then
   bytes=4
   binary=$(printf "%021d" $binary)
   read -d=' ' -a binary_arr < <(sed 's/./ &/g' <<< "$binary")
   binary_arr[0]="11110${binary_arr[0]}"
   binary_arr[3]="10${binary_arr[3]}"
   binary_arr[9]="10${binary_arr[9]}"
   binary_arr[15]="10${binary_arr[15]}"
  fi 
  printf "${bytes} Byte unicode: \U${codepoint}\n"
  echo "Codepoint: U${codepoint}"
  echo "Codepoint binary: $binary"
  binary=$(sed 's/ //g' <<< "${binary_arr[@]}")
  echo "UTF-8 binary: ${binary}"
  echo "UTF-8 hex: $(printf "%X" $((2#${binary})))"
}
$>cp2hex U0024
1 Byte unicode: $
Codepoint: U0024
Codepoint binary: 00100100
UTF-8 binary: 00100100
UTF-8 hex: 24

$>cp2hex U00A2
2 Byte unicode: ¢
Codepoint: U00A2
Codepoint binary: 00010100010
UTF-8 binary: 1100001010100010
UTF-8 hex: C2A2

$>cp2hex 20AC
3 Byte unicode: €
Codepoint: U20AC
Codepoint binary: 0010000010101100
UTF-8 binary: 111000101000001010101100
UTF-8 hex: E282AC

$>cp2hex U1F604
4 Byte unicode: 😄
Codepoint: U1F604
Codepoint binary: 000011111011000000100
UTF-8 binary: 11110000100111111001100010000100
UTF-8 hex: F09F9884

Friday, May 31, 2024

Powershell Function to List Ports in Use by a Java Prog

Replace"YourCurrentProject" default with your most frequent query.

Add the following to your $profile (PS> code $profile).

function lsAppPorts($prog = "YourCurrentProject") {
    foreach($a in jps -l | C:\Progra~2\GnuWin32\bin\awk "/$prog/{ print `$1; }") {
        netstat -ano | findstr $a
    }
}

Note: Requires Gnu Awk for Windows

Thursday, May 2, 2024

Generate Tough Passwords From the Commandline

Note: Requires openssl
Usage: $> genpasswd [int size, default: 8]
alias genpasswd='_(){ if (($#<1)); then PASSWDSIZE=8; else PASSWDSIZE=${1};fi; openssl rand -base64 32|sed -r "s/^(.{${PASSWDSIZE}})(.*)/\1/g"; }; _'

Thursday, February 15, 2024

Custom SpringBoot Serializer for REST Controller - Fairly Elegant Way

SpringBoot is great for speed of development when you are familiar with it. Modern development often is centered on microservices which is heavily reliant upon a REST controller that returns JSON formatted data.

At some point there is a high chance that you'll need to customize the Serialization of data returned from the REST controller.

At first glance it seems straightforward enough, just define your custom serializer and reference it within in the @JsonSerializer annotation: @JsonSerialize(using = MyCustomSerializer.class)

If you do not register your custom Serializer you will likely get a 500 error stating that a serializer could not be found.

You must also make sure that you define a handledType() in your custom Serializer. If not you will see a message along the lines of:
... does not define valid handledType()

Furthermore you cannot use @JsonSerializer(using = ...) in your REST controller where your Serializer is defined, for that you need a custom annotation.

Most tutorials online I've found have circumvented these steps and provided kludgey solutions reliant on creation of another class - unregistered serializers are found when a wrapper class for instance is utilized defined within the REST controller.

Of course, you would probably put your custom Serializer somewhere outside of your REST controller, but for simplicity sake this example has it all in one.

import ... 
...
@RestController
@Component // Make sure our @Bean is found 
... 

////
// Define our Serializer with handler 
////
public class MyLongSerializer extends JsonSerializer<Long> {
    private static final long serialVersionUID = -8885817712043352432L;

    @Override
    public void serialize(Long value, JsonGenerator jsonGenerator, SerializerProvider serializerProvider)
            throws IOException {
    	jsonGenerator.writeStartObject();
        jsonGenerator.writeNumberField("my val", value);
        jsonGenerator.writeEndObject();
    }
    
    @Override
    public Class<Long> handledType() {
	return Long.class;
    }

}

////
// Define our custom Serializer annotation - @MyLong
////
@Retention(RetentionPolicy.RUNTIME)
@JacksonAnnotationsInside
@JsonInclude(value = Include.NON_NULL)
@JsonSerialize(using = MyLongSerializer.class)
public @interface MyLong {
}

////
// Define our REST controller test 
////
@RequestMapping(value = { "/testLong" }, method = RequestMethod.GET, produces = {
		MediaType.APPLICATION_JSON_VALUE }, consumes = MediaType.ALL_VALUE)
public Long testLong() { 
    @MyLong
    Long myLong = 293874928749284L;
	return myLong;
}

////
// Add our dear custom serializers
////
@Bean
public Jackson2ObjectMapperBuilder jackson2ObjectMapperBuilder() {
    SimpleModule m = new SimpleModule();
    m.addSerializer(new MyLongSerializer());
    // Add here as many as you have: 
     // m.addSerializer(...); ... 
    return new Jackson2ObjectMapperBuilder().modules(m);
}	

Hit your url: http://.../testLong and you should see:
{
"my val": 293874928749284
}

Wednesday, March 30, 2022

Defaulting Windows loopback (localhost) to IPv4 (127.0.0.1) (from the IPv6 (::1) default)

For a while now (since 8.1) unbeknownst to most people MS has moved localhost resolution into the DNS stack itself. As a result, overriding the mapping for localhost from ::1 to 127.0.0.1 can no longer be done via the hosts file. 

Here's the magic one-liner (run from an elevated prompt): 

netsh interface ipv6 add prefixpolicy ::ffff:0:0/96 precedence=51 label=4 store=persistent 

Ping localhost and voila it should return 127.0.0.1. You can play around with the precedence to see it in action, first check the existing precedences netsh interface ipv6 show prefixpolicies you want to check the precedence of ::1/128 that's the prefixpolicy whose precedence needs to be beat by our new one. Set it below ping localhost then above. 


Some recommend you issue a netsh interface ipv6 reset but in my experience on Windows 11 Pro it's not necessary. 

Thanks to MatrixPost Blog for the mapping: Prefer IPv4 over IPv6 in Windows Networks - .matrixpost.net

Sunday, March 22, 2020

COVID-19 Data Aggregator

#!/bin/bash 
  ######################################################################
 ######################################################################
## Author: A.M. Danischewski, Created Date: 20200322, Version: v1.01
## File ID: b7c4a0cb19767d4d88f1cbd49474dad4
## Last Modified: 2020-03-22
## Issues: If you find any issues emai1 me at my <first name> dot 
##         <my last name> at gmail dot com.  
##
## This script fetches from arcgis.com COVID-19 data by default every 10 
## minutes.  
## 
## This program is free software: you can redistribute it and/or modify
## it under the terms of the GNU General Public License as published by
## the Free Software Foundation, either version 3 of the License, or
## (at your option) any later version.
##
## This program is distributed in the hope that it will be useful,
## but WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
## GNU General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with this program. If not, see <http://www.gnu.org/licenses/>.
 ######################################################################
  ######################################################################


declare URL='https://services1.arcgis.com/0MSEUqKaxRlEPj5g/arcgis/rest/services/ncov_cases/FeatureServer/1/query?f=json&where=(Confirmed%20%3E%200)%20AND%20(Deaths%3E0)&returnGeometry=false&spatialRel=esriSpatialRelIntersects&outFields=*&orderByFields=Deaths%20desc%2CCountry_Region%20asc%2CProvince_State%20asc&outSR=102100&resultOffset=0&resultRecordCount=250&cacheHint=true' 
 ## Fetch Interval (600 = 10 minutes) 
declare SLEEP=600 
 ## Change DATA_DIR to a writable location where you want keep your data 
 ## eg. /mnt/hyper1/covid19
declare DATA_DIR="/home/${USER}/covid19" 
declare FILE_PFX="covid_data_" 
declare DATE_SFX="" 
declare FILE_EXT="txt" 
declare FILE_NAME="" 

mkdir -p "${DATA_DIR}" 

while :; do 
 DATE_SFX=$(date +"%m%d%Y%H%M")
 FILE_NAME="${FILE_PFX}${DATE_SFX}.${FILE_EXT}"
 curl -S "$URL" > "${DATA_DIR%%/}/${FILE_NAME}"
 sleep ${SLEEP}
done 

exit 0
Get the latest on GitHub.

Tuesday, December 24, 2019

Uncomment - Quickly check config files

alias uncomment='_(){ if (($#<1))||[[ "${1}" =~ ^- || ! -e "${1}" ]]; then (($#>0))&&[[ ! "${1}" =~ ^- ]]&&[[ ! -e "$1" ]]&&echo "*** Error: File (\"$1\") not found.";echo "Usage: uncomment <file>"; return 1;fi;grep -iP "^([ \t\r\f]|[^#])*[^# \h\t\f\r]+" "$1";};_' 
Get the latest on GitHub.

Monday, November 11, 2019

Custom File Colorer - In AWK

#!/usr/bin/awk -f

  ######################################################################
 ######################################################################
## Author: Adam Michael Danischewski 
## GitHub: https://github.com/AdamDanischewski/scriptsandoneliners
## Created Date: 2019-11-12
## Name: colorfiles.awk
## Version: v0.00
## Last Modified: 2019-11-12
## Issues: If you find any issues emai1 me at <my first name> (dot) 
##         <my last name> (at) gmail (dot) com. 
##
## Requirements: awk 
## 
## Allows the user to fine tune and customize their color scheme for 
## files. LS_COLORS only allows for a small subset of file types. This 
## script allows specific colors for all files and can be utilized in 
## pipelines to color output after processing with tools that do not 
## have a color option. 
## 
## This script includes a color template that can be applied to 
## pipelines on the command line or to replace coloring features 
## of shell utilities. Edit/hack to your preference. 
## 
## This program is distributed in the hope that it will be useful,
## but WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  
##
## Released under Creative Commons License: CC BY-SA 4.0
## https://creativecommons.org/licenses/by-sa/4.0/
 ######################################################################
  ######################################################################

function init_colormap() {
 colormap["pdf"]="108;107;21"
 startcolor="\033[38;2;"         ## TrueColor RGB triplet prefix 
 endcolor="\033[0m"
 colormap["jpg"]="0;85;0"  
 colormap["gif"]=colormap["jpg"] ## Grouping images, change as you wish.
 colormap["png"]=colormap["jpg"]
 colormap["jpeg"]=colormap["jpg"] 
 colormap["jpe"]=colormap["jpg"] ## JFIF
 colormap["blend"]="251;143;21"
 colormap["mp3"]="157;0;193"
 colormap["bsh"]="85;229;0"
 colormap["awk"]=colormap["bsh"]
 colormap["php"]=colormap["bsh"]
 colormap["js"]=colormap["bsh"]
 colormap["pl"]=colormap["bsh"]
 colormap["py"]=colormap["bsh"]
 colormap["vtt"]="0;85;157"  
 colormap["tex"]="0;0;229"  
 colormap["nzb"]="85;229;0"  
 colormap["deb"]="193;12;199"  
 colormap["txt"]="215;215;21"  
 colormap["mp4"]="85;0;193"  
 colormap["log"]="193;85;12"  
 colormap["c"]="0;121;229"  
 colormap["cc"]=colormap["c"]
 colormap["cpp"]=colormap["c"]
 colormap["h"]=colormap["c"]
 colormap["avi"]=colormap["mp4"]
 colormap["mov"]=colormap["mp4"]
 colormap["mkv"]=colormap["mp4"] 
 colormap["webm"]=colormap["mp4"] 
 colormap["html"]="157;0;193"
 colormap["css"]=colormap["html"]
 colormap["pdf"]="229;85;0"
 colormap["dir"]="0;157;229"    ## Directories - ending in / 
 colormap["links"]="251;22;143" ## Symbolic links - ending in @
 colormap["part"]="157;0;85"
 colormap["tar"]="229;0;193"
 colormap["tar.gz"]=colormap["tar"]
 colormap["tgz"]=colormap["tar"]
 colormap["tar.xz"]=colormap["tar"]
 colormap["tar.bz2"]=colormap["tar"]
 colormap["xz"]="229;0;85"
 colormap["gz"]=colormap["xz"]
 colormap["7z"]=colormap["xz"]
 colormap["iso"]=colormap["xz"]
 colormap["rar"]=colormap["xz"]
 colormap["par2"]=colormap["xz"]
 colormap["bz2"]=colormap["xz"]
 colormap["zip"]=colormap["xz"]
 colormap["kgb"]=colormap["xz"] # Future standard compression (PAQ)
 return
} 

function wrap_color(s) { 
 if (tolower(s) ~ /\.jpg[*]?$/) { 
  selc=colormap["jpg"]
 } else if (tolower(s) ~ /\.jpe[*]?$/) { 
  selc=colormap["jpe"]  
 } else if (tolower(s) ~ /\.gif[*]?$/) { 
  selc=colormap["gif"]  
 } else if (tolower(s) ~ /\.tar[*]?$/)    { 
  selc=colormap["tar"]  
 } else if (tolower(s) ~ /\.tar\.gz[*]?$/) { 
  ## Make sure these are before their respective 
  ## compression conditionals to avoid tar's 
  ## from being treated as simply compressed. 
  selc=colormap["tar.gz"]  
 } else if (tolower(s) ~ /\.tar.bz2[*]?$/)  { 
  selc=colormap["tar.bz2"]  
 } else if (tolower(s) ~ /\.tar.xz[*]?$/)  { 
  selc=colormap["tar.xz"]  
 } else if (tolower(s) ~ /\.tgz[*]?$/)  { 
  selc=colormap["tgz"]  
 } else if (tolower(s) ~ /\.jpeg[*]?$/) { 
  selc=colormap["jpeg"]  
 } else if (tolower(s) ~ /\.png[*]?$/)   { 
  selc=colormap["png"]               
 } else if (tolower(s) ~ /\.mp4[*]?$/)   { 
  selc=colormap["mp4"]              
 } else if (tolower(s) ~ /\.rar[*]?$/)   { 
  selc=colormap["rar"]         
 } else if (tolower(s) ~ /\.par2[*]?$/)   { 
  selc=colormap["par2"]         
 } else if (tolower(s) ~ /\.log[*]?$/)   { 
  selc=colormap["log"]         
 } else if (tolower(s) ~ /\.mp3[*]?$/)   { 
  selc=colormap["mp3"]         
 } else if (tolower(s) ~ /\.deb[*]?$/)   { 
  selc=colormap["deb"]  
 } else if (tolower(s) ~ /\.mov[*]?$/)   { 
  selc=colormap["mov"]  
 } else if (tolower(s) ~ /\.bsh[*]?$/)   { 
  selc=colormap["bsh"]  
 } else if (tolower(s) ~ /\.mkv[*]?$/)   { 
  selc=colormap["mkv"]  
 } else if (tolower(s) ~ /\.pdf[*]?$/)   { 
  selc=colormap["pdf"]  
 } else if (tolower(s) ~ /\.c[*]?$/)   { 
  selc=colormap["c"]  
 } else if (tolower(s) ~ /\.cpp[*]?$/)   { 
  selc=colormap["cpp"]  
 } else if (tolower(s) ~ /\.cc[*]?$/)   { 
  selc=colormap["cc"]  
 } else if (tolower(s) ~ /\.h[*]?$/)   { 
  selc=colormap["h"]  
 } else if (tolower(s) ~ /\.txt[*]?$/)   { 
  selc=colormap["txt"]  
 } else if (tolower(s) ~ /\.vtt[*]?$/)   { 
  selc=colormap["vtt"]  
 } else if (tolower(s) ~ /\.tex[*]?$/)   { 
  selc=colormap["tex"]  
 } else if (tolower(s) ~ /\.nzb[*]?$/)  { 
  selc=colormap["nzb"]  
 } else if (tolower(s) ~ /\.bz2[*]?$/)  { 
  selc=colormap["bz2"]  
 } else if (tolower(s) ~ /\.iso[*]?$/)  { 
  selc=colormap["iso"]  
 } else if (tolower(s) ~ /\.html[*]?$/)  { 
  selc=colormap["html"]  
 } else if (tolower(s) ~ /\.css[*]?$/)  { 
  selc=colormap["css"]  
 } else if (tolower(s) ~ /@$/)  { 
  selc=colormap["links"]  
 } else if (tolower(s) ~ /\.webm[*]?$/)  { 
  selc=colormap["webm"]  
 } else if (tolower(s) ~ /\.awk[*]?$/)    { 
  selc=colormap["awk"]  
 } else if (tolower(s) ~ /\.php[*]?$/)    { 
  selc=colormap["php"]  
 } else if (tolower(s) ~ /\.js[*]?$/)    { 
  selc=colormap["js"]  
 } else if (tolower(s) ~ /\.pl[*]?$/)    { 
  selc=colormap["pl"]  
 } else if (tolower(s) ~ /\.py[*]?$/)    { 
  selc=colormap["py"]  
 } else if (tolower(s) ~ /\.part[*]?$/)    { 
  selc=colormap["part"]  
 } else if (tolower(s) ~ /\.kgb[*]?$/)    { 
  selc=colormap["kgb"]  
 } else if (tolower(s) ~ /\.xz[*]?$/)  { 
  selc=colormap["xz"]  
 } else if (tolower(s) ~ /\.7z[*]?$/)  { 
  selc=colormap["7z"]  
 } else if (tolower(s) ~ /\.gz[*]?$/)  { 
  selc=colormap["gz"]  
 } else if (tolower(s) ~ /\.zip[*]?$/)  { 
  selc=colormap["zip"]  
 } else if (tolower(s) ~ /\/$/)  { 
  selc=colormap["dir"]  
 }
 else { 
  return s
 }  
 return (startcolor selc "m" s endcolor)
}

function usage() {
 usage_str=("\nUsage: /bin/ls -f1 | colorfiles.awk \n\n \
   Options: \n\
             -?    Help.\n\n\
Note: Make sure colors are not already present- you can sed the input.\n\
      Eg. $> ls --color| sed 's/\\x1B[[0-9;]\\+[A-Za-z]//g' |colorfiles.awk\n\
          $> alias colorfiles='sed \"s/\x1B[[0-9;]\\+[A-Za-z]//g\"|colorfiles.awk'\n\
          $> alias lsd='find \"$(pwd)\" -maxdepth 1 -type d -printf \"%T@\t%Tc %f/\\n\" 2>/dev/null|sort -n|cut -f 2-|tail -50|colorfiles'\n\
          $> alias lsf='find \"$(pwd)\" -maxdepth 1 -type f -printf \"%T@\t%Tc %f\\n\" 2>/dev/null|sort -n|cut -f 2-|tail -50|colorfiles'\n\
          $> alias ls='/bin/ls -F -f1|colorfiles'\n")
 return usage_str
}

BEGIN {if(v_opt1=="-?"){print usage();exit 1;}else{init_colormap();};}

## Main
{
 print wrap_color($0) 
}

Get the latest on GitHub.