A Golang program to dump serial data into CSV file

A unique situation in which I wanted to dump the memory of a program running on a microcontroller – the program can send data through serial port, but for it to make sense for the programmer it has to be dumped in a readable format. And another challenge was that the controller was programmable only from MS Windows. So this dumping program must be able to run on Win64.

I chose Golang for this purpose as I don’t know the serial port reading API of Win32/64 but I can build for Win64 using Golang on Linux and a nice Serial library was available for Golang as well.

package main

import (
	"fmt"
	"github.com/tarm/serial"
	"bufio"
	"os"
	"strconv"
	"time"
	"encoding/hex"
	"bytes"
	"encoding/binary"
	"sync"
)

type mt_var struct {
	u32 uint32
	i32 int32
	f32 float32
	choice int64
}

func (v *mt_var) Parse(varvalue_b []byte) {
	reader := bytes.NewReader(varvalue_b)

	switch v.choice {
	case 1:
		if err := binary.Read(reader, binary.BigEndian, &v.u32); err != nil {
			fmt.Println("Error occurred while parsing bytes to uint32: ", err)
		}
	case 2:
		if err := binary.Read(reader, binary.BigEndian, &v.i32); err != nil {
			fmt.Println("Error occurred while parsing bytes to int32: ", err)
		}
	case 3:
		if err := binary.Read(reader, binary.BigEndian, &v.f32); err != nil {
			fmt.Println("Error occurred while parsing bytes to float: ", err)
		}
	}
}

func (v mt_var) String() (ret string) {
	switch v.choice {
	case 1:
		ret = fmt.Sprintf("%v", v.u32)
	case 2:
		ret = fmt.Sprintf("%v", v.i32)
	case 3:
		ret = fmt.Sprintf("%v", v.f32)
	}
	return
}

func main() {
	scanner := bufio.NewScanner(os.Stdin)

	defer func() {
		fmt.Println("Press any key to exit")
		scanner.Scan()
	}()

	fmt.Print("Enter serial port name: ")
	scanner.Scan()
	portname := scanner.Text()

	fmt.Print("Enter baud rate: ")
	scanner.Scan()
	baud_s := scanner.Text()
	baud, err := strconv.ParseInt(baud_s, 10, 64)

	if err != nil {
		fmt.Println("Unable to parse baud rate: ", err)
		return
	}

	fmt.Print("Enter serial port timeout in seconds: ")
	scanner.Scan()
	timeout_s := scanner.Text()
	timeout, err := strconv.ParseInt(timeout_s, 10, 64)

	if err != nil {
		fmt.Println("Unable to parse timeout value: ", err)
		return
	}

	fmt.Print("Select data type of variables (1 - uint32, 2 - int32, 3 - float32): ")
	scanner.Scan()
	dt_var_s := scanner.Text()
	dt_var, err := strconv.ParseInt(dt_var_s, 10, 64)

	if err != nil {
		fmt.Println("Unable to parse choice value for data type: ", err)
		return
	} else if dt_var < 1 || dt_var > 3 {
		fmt.Println("Invalid choice ", dt_var, " for data type selection")
		return
	}

	fmt.Print("Enter number of variables to be read per timestamp: ")
	scanner.Scan()
	num_vars_s := scanner.Text()
	num_vars, err := strconv.ParseInt(num_vars_s, 10, 64)

	if err != nil {
		fmt.Println("Unable to parse num vars value: ", err)
		return
	}

	fmt.Print("Enter number of bytes per variable: ")
	scanner.Scan()
	num_bytes_per_var_s := scanner.Text()
	num_bytes_per_var, err := strconv.ParseInt(num_bytes_per_var_s, 10, 64)

	if err != nil {
		fmt.Println("Unable to parse num bytes per var value: ", err)
		return
	}

	fmt.Print("Enter total number of timestamps: ")
	scanner.Scan()
	num_timestamps_s := scanner.Text()
	num_timestamps, err := strconv.ParseInt(num_timestamps_s, 10, 64)

	if err != nil{
		fmt.Println("Unable to parse num timestamps value: ", err)
		return
	}

	fmt.Print("Enter DAQ sample time: ")
	scanner.Scan()
	sampletime_s := scanner.Text()
	sampletime, err := strconv.ParseInt(sampletime_s, 10, 64)

	if err != nil {
		fmt.Println("Unable to parse sample time value: ", err)
		return
	}

	c := &serial.Config{Name: portname, Baud: int(baud), ReadTimeout: time.Second * time.Duration(timeout)}
	port, err := serial.OpenPort(c)

	if err != nil {
		fmt.Println("Unable to open serial port: ", err)
		return
	}

	defer func() {
		fmt.Println("Closing serial port")
		port.Close()
	}()

	var wg sync.WaitGroup
	var stop_serial_writer = make(chan bool)

	fmt.Println("Press any key to start reading serial port")
	scanner.Scan()

	wg.Add(1)
	go serial_writer(port, stop_serial_writer, &wg)

	fmt.Println("Starting to read port")

	writer_channel := make(chan []byte, 1000)

	wg.Add(1)
	go writer(writer_channel, num_vars, &wg, sampletime, num_bytes_per_var, dt_var)

	bytes_per_timestamp := num_vars * num_bytes_per_var
	var bytecount int

outer:
	for i := int64(1); i <= num_timestamps; i++ {
		buf := make([]byte, bytes_per_timestamp)

		for j := int64(0); j < bytes_per_timestamp; j++ {
			if n, err := port.Read(buf[j:j+1]); err != nil {
				fmt.Println("Error occurred ", err)
				break outer
			} else {
				bytecount += n
				fmt.Printf("Read %d bytes\r", bytecount)
			}
		}

		writer_channel <- buf
	}

	fmt.Println()

	close(writer_channel)
	stop_serial_writer <- true

	fmt.Println("waiting for all goroutines to exit")
	wg.Wait()
}

func writer(channel chan []byte, num_vars int64, wg *sync.WaitGroup, sampletime int64, bytes_per_var int64, dt_var int64) {
	defer func() {
		wg.Done()
	}()

	home_directory, err := os.UserHomeDir()
	if err != nil {
		fmt.Println("Unable to fetch user home directory: ", err)
		return
	}

	time_str := time.Now().Format("2006-01-02-15-04-05")
	filename_raw := fmt.Sprintf("%s%cserial2csv_raw_%s.txt", home_directory, os.PathSeparator, time_str)
	filename_csv := fmt.Sprintf("%s%cserial2csv_%s.csv", home_directory, os.PathSeparator, time_str)

	f_raw, err := os.Create(filename_raw)
	if err != nil {
		fmt.Println("Unable to create file ", filename_raw, ": ", err)
		return
	}
	defer f_raw.Close()

	f_csv, err := os.Create(filename_csv)
	if err != nil {
		fmt.Println("Unable to create file ", filename_csv, ": ", err)
		return
	}
	defer f_csv.Close()

	bytes_written := 0
	var o_sampletime int64


	for data := range channel {
		hexdata := make([]byte, hex.EncodedLen(len(data)))
		hex.Encode(hexdata, data)
		if n, err := f_raw.Write(hexdata); err != nil {
			bytes_written += n
			fmt.Println("Error occurred while writing to file ", filename_raw, ": ", err)
		}

		fmt.Fprintf(f_csv, "%v,", o_sampletime)
		o_sampletime += sampletime

		for varnumber := int64(1); varnumber <= num_vars; varnumber++ {
			var err error
			var n int

			varvalue_b := data[varnumber * bytes_per_var - bytes_per_var:varnumber * bytes_per_var]
			var varvalue = mt_var{choice: dt_var}
			varvalue.Parse(varvalue_b)

			if varnumber == num_vars {
				n, err = fmt.Fprintln(f_csv, varvalue)
			} else {
				n, err = fmt.Fprintf(f_csv, "%v,", varvalue)
			}

			if err != nil {
				fmt.Println("Error while writing to file ", filename_csv, ": ", err)
			} else {
				bytes_written += n
			}
		}
	}

	fmt.Println("Total bytes written to files ", bytes_written)
	fmt.Println("RAW Data: ", filename_raw)
	fmt.Println("CSV Data: ", filename_csv)
}

func serial_writer(port *serial.Port, stop chan bool, wg *sync.WaitGroup) {
	fmt.Println("Starting serial port writer - sending 1 continuously")

	defer wg.Done()

	data := []byte{1}
	zero := []byte{0}

outer_one:
	for {
		select {
		case <-stop:
			fmt.Println("stopping to send 1 on serial")
			break outer_one
		default:
			port.Write(data)
		}
	}

	zero_stop_timer := time.NewTimer(2 * time.Second)

outer_zero:
	for {
		select {
		case <-zero_stop_timer.C:
			fmt.Println("stopping to send 0 on serial")
			break outer_zero
		default:
			port.Write(zero)
		}
	}

	zero_stop_timer.Stop()
}

The same is available on my Github.

Building this program on Linux, for Win64 is pretty easy (my Linux box is amd64):

GOOS=windows go build serial2csv_sync_mdt.go

You may require some changes in the build command if building for 32 bit or from a 32 bit system.

Golang on OpenWRT MIPS

I have been tracking Golang for quite a while since I came to know about it I guess about 3 years ago primarily because it is very easy to use and build static binaries that just work about anywhere. And no dealing with memory allocation stuff which often lead to frustrations and segmentation fault bugs soaking up hours of your time to solve those.

As a OpenWRT user running a Go program on OpenWRT had been one of my most desired things. So here it is, finally, a hello world program running my TP Link WR740N (which is a MIPS 32 bit CPU, ar71xx in OpenWRT tree):

system type             : Atheros AR9330 rev 1
machine                 : TP-LINK TL-WR741ND v4
processor               : 0
cpu model               : MIPS 24Kc V7.4
BogoMIPS                : 265.42
wait instruction        : yes
microsecond timers      : yes
tlb_entries             : 16
extra interrupt vector  : yes
hardware watchpoint     : yes, count: 4, address/irw mask: [0x0ffc, 0x0ffc, 0x0ffb, 0x0ffb]
isa                     : mips1 mips2 mips32r1 mips32r2
ASEs implemented        : mips16
shadow register sets    : 1
kscratch registers      : 0
package                 : 0
core                    : 0
VCED exceptions         : not available
VCEI exceptions         : not available
package main

import "fmt"

func main() {
        fmt.Println("hello world")
}

First I built it with GOOS=linux GOARCH=mips go build hello but it did not run and gave error “Illegal Instruction”. Then I tried it with GOOS=linux GOARCH=mipsle go build hello which again, did not work because the CPU of this TP Link is big endian, not little endian. After a bit of searching I came across this GoMips guide on Golang’s Github which builds it using GOMIPS=softfloat. I tried the same and my program works! It will now be easy to build complex stuff that runs on embedded devices without resorting to C/C++.

$ GOOS=linux GOARCH=mips GOMIPS=softfloat go build hello
$ scp hello root@IP:/tmp/

root@740n-2:/tmp# ./hello
hello world
root@740n-2:/tmp#                                                                                        

Change username and hostname for Ubuntu instances on AWS

If you have used Ubuntu images on AWS, you might have noticed that the default username of the user on the instance is ‘ubuntu’. And the hostname is dynamically generated according to the public IP. Both of these can be changed using cloud-config supported on Ubuntu images – the config has to be provided in the User Data section in Advanced on the Configure Instance tab.

YAML configuration to change the parameters:

#cloud-config
fqdn: myhostname
system_info:
  default_user:
    name: myusername

A lot more things are possible using the cloud-config method and it is supported on other operating system images as well such as CentOS. Take a look at Cloud config examples.

Using privileged mode (become) in Ansible without a password

So I was working on automating some stuff using Ansible when the necessity to have password less superuser access came up. A simple way would be adding the ansible management key to the root account itself and allow SSH to root, but allowing ssh to root is usually a bad idea.

I tried many things – NOPASSWD in sudo entry, requiretty, etc. And after nearly two hours of digging a spark ignited and I found a way – Linux has PAM module called pam_wheel.so which can implicitly allow root access via su when a user is present in the wheel group (the group can be configured in module options). This module is disabled by default on most Linux distributions, in fact Ubuntu doesn’t even have a wheel group. But in this particular case I was managing CentOS which has the wheel group.

Add the Ansible management user to the wheel group and enable the pam_wheel.so module:

# /etc/pam.d/su

# Uncomment the following line to implicitly trust users in the "wheel" group.
auth            sufficient      pam_wheel.so trust use_uid

Now when you SSH to the machine using the ansible user and run su – it will give you root access without asking for password. Consequently, now when you set become_method = su in your Ansible configuration by way of editing config files, setting variables in playbook or inventory, etc. Ansible will become privileged without a password.

LXD OpenVSwitch and VLANs

LXD is a fantastic container virtualization tool that comes by default with Ubuntu. In one of my applications I needed to have many containers each within it’s own VLAN network.
So I used OpenVSwitch in combination with LXD to achieve this.

There is no inherent facility in LXD to provide VLAN tag numbers to the interface. So it is necessary to use a “Fake bridge”. I managed to do it after reading this article by Scott – VLANs with Open vSwitch Fake Bridges

Let’s say the OpenVSwitch bridge is named vm-bridge and we want to add 10 fake bridges ranging from VLAN 20 to 30. Here’s how I did it:

for i in $(seq 20 30); do
ovs-vsctl add-br vlan$i vm-bridge $i
done

In LXD you can specify the bridge to which it will connect containers to, so I created 10 containers using a similar loop 😀
Further to bind each container to the fake bridge this step is needed:

for i in $(seq 20 30); do
lxc config device set ct$i eth0 parent vlan$i
done

You can no longer count on reliability of budget smartphones or Android One

The story here is about my bad experience with a Nokia 7 Plus smartphone which is a certified Android One device.

I have been using Android over the last 7-8 years or so, and like every geek out there I was involved in flashing custom ROMs and tracking XDA forums for new builds. I even had one of the best phones suited for this purpose – The LG Nexus 4.

Read More

A small review of JBL E65BTNC

In 2016, I came across a nice deal for an on-ear headphone – The Motorola Tracks Air. It was selling on Flipkart for ₹2500. That was steal deal, considering the original price is ₹8990. I used it for on and off for quite some time but the on-ear type meant it started hurting my ears when using them for more than 30 minutes. So gradually the usage waned off and I stopped using it. Usually I do not buy new stuff unless the previous one I have is completely dead, this is especially true in case of electronics. Because of the online shopping deals it’s very easy to accumulate unnecessary junk. Just like that due to impulsive purchases without much thought I have a few electronic junk lying around which is in pristine condition not used even a single time.

In order to get rid of the Tracks Air headphones which I had, I tried putting an ad for it on OLX India site which is a famous marketplace for pre-owned stuff. I have successfully sold quite a lot of things on the platform and even bought a few. But for some reason people didn’t seem to be interested in this headphone at any price, so I gave it to someone I knew and didn’t have any headphone for free. At least something lying in my junk is useful to someone.

Read More

Maintain lead acid batteries regularly

Thursdays are usually maintenance day for the electrical power supply company in my area. So there was nearly a full day power cut. Luckily, I have a UPS so that sorts out the problem for 8-9 hours. The lead acid battery I use for my UPS is about 3-4 years old, and these being unsealed batteries they last long, really long if maintained properly.

In the past I have had one such battery last for a decade before requiring a replacement.

Unsealed lead acid batteries require two important maintenance activities:

  1. Topping up distilled water every 6 months
  2. Applying petroleum jelly / grease on the terminals to prevent corrosion
Read More

Ubuntu 18.04 add e1000e Intel driver to dkms

Note: The compile process appears to be broken for driver version 3.8.4. So the following steps will not work for that version. This post will be updated when a suitable fix is found for the same.

Here’s a quick guide on how to add the Intel e1000e driver to DKMS (Dynamic Kernel Module Support) so that it gets installed / uninstalled automatically with future kernel updates and removals.

Download the driver from Intel website https://downloadcenter.intel.com/download/15817

As of my writing this article, the e1000e version is 3.4.2.1. On download the tarball I get e1000e-3.4.2.1.tar.gz.

Extract it to /usr/src:

tar -xzf e1000e-3.4.2.1.tar.gz -C /usr/src

Create a dkms.conf in /usr/src/e1000e-3.4.2.1 with following contents:

PACKAGE_NAME="e1000e"
PACKAGE_VERSION="3.4.2.1"
AUTOINSTALL=yes
MAKE[0]="make -C src/"
BUILT_MODULE_NAME="e1000e"
BUILT_MODULE_LOCATION="src/"
DEST_MODULE_LOCATION="/kernel/drivers/net/ethernet/intel/e1000e"

Next, we have to tell DKMS that such a module has been added and build it for each of the kernels we have on the system:

dkms add -m e1000e/3.4.2.1
for k in /boot/vmlinuz*; do
  dkms install -k ${k##*vmlinuz-} e1000e/3.4.2.1
done

Finally, reboot the system and the new module should be live.

Date range in a MariaDB query using the Sequence Engine

One of my applications involved generating a date-wise report for items created on that day and we needed zeroes against the count of items on the date which had no entries.

User selects a date range and the application must generate this report. Not so easy if I had not come across the MariaDB Sequence Storage Engine!

Sequences have long been good features in databases like Oracle, PostgreSQL and the likes, I absolutely had no idea of it’s existence in MariaDB — just came across it while browsing the documentation of MariaDB.

Here’s a sample of my use case:

MariaDB [test]> create table items (id int unsigned primary key auto_increment, date_created datetime not null);
Query OK, 0 rows affected (0.061 sec)

MariaDB [test]> insert into items (date_created) values ('2019-01-01'), ('2019-01-05'), ('2019-01-06'), ('2019-01-06'), ('2019-01-01'), ('2019-01-10'), ('2019-01-09'), ('2019-01-09'), ('2019-01-09');
Query OK, 9 rows affected (0.032 sec)
Records: 9  Duplicates: 0  Warnings: 0

MariaDB [test]> select * from items;
+----+---------------------+
| id | date_created        |
+----+---------------------+
|  1 | 2019-01-01 00:00:00 |
|  2 | 2019-01-05 00:00:00 |
|  3 | 2019-01-06 00:00:00 |
|  4 | 2019-01-06 00:00:00 |
|  5 | 2019-01-01 00:00:00 |
|  6 | 2019-01-10 00:00:00 |
|  7 | 2019-01-09 00:00:00 |
|  8 | 2019-01-09 00:00:00 |
|  9 | 2019-01-09 00:00:00 |
+----+---------------------+
9 rows in set (0.001 sec)

MariaDB [test]> select date(date_created), count(id) from items group by date(date_created);
+--------------------+-----------+
| date(date_created) | count(id) |
+--------------------+-----------+
| 2019-01-01         |         2 |
| 2019-01-05         |         1 |
| 2019-01-06         |         2 |
| 2019-01-09         |         3 |
| 2019-01-10         |         1 |
+--------------------+-----------+
5 rows in set (0.001 sec)

MariaDB [test]> 

After a couple of attempts with the samples provided in the MariaDB documentation page, I managed to devise a query which provided me exactly what I needed, using SQL UNION:

MariaDB [test]> select dt, max(cnt) from ( select cast( date_add('2019-01-01', interval seq day) as date ) dt, 0 cnt from seq_0_to_11 union select cast( date(date_created) as date ) dt, count(id) cnt from items where date(date_created) between '2019-01-01' and '2019-01-11' group by date(date_created) ) t group by dt order by dt;
+------------+----------+
| dt         | max(cnt) |
+------------+----------+
| 2019-01-01 |        2 |
| 2019-01-02 |        0 |
| 2019-01-03 |        0 |
| 2019-01-04 |        0 |
| 2019-01-05 |        1 |
| 2019-01-06 |        2 |
| 2019-01-07 |        0 |
| 2019-01-08 |        0 |
| 2019-01-09 |        3 |
| 2019-01-10 |        1 |
| 2019-01-11 |        0 |
| 2019-01-12 |        0 |
+------------+----------+
12 rows in set (0.001 sec)

Yeah, that’s basically filling in zero values for the dates on which there were no entries. Can this be done using RIGHT JOIN? I tried to but couldn’t form a JOIN condition. If you know drop a comment!