Logo
RSS Feed

📘 Linux BTFM


Created: 23.09.2020

Variables

# no spaces when assigning
$0 # the first arg

# array
array=(1 2 4 5)
${array[0]}
${array[*]} # all items delimited by IFS
${array[@]} # all items 
${!array[@]} # all indexes
${#array[@]} # number of items
${#array[0]} # length of item 0

ARRAY=($(ls *.txt))
COUNT=1
for FILE in "${ARRAY[@]}"
do
	echo -n $FILE
	echo -n "[${ARRAY[$COUNT]}]"
	if [ -w "$FILE"]; then
		echo -e "\t YES"
	else
		echo -e "\t NO"
done

Functions

function functionname	() {
	local name=$1
	commands
}
hello

function functioname2 {
 commands		
}

Conditions

# if/elif
if [ $# -ne 1]; then
  echo "Exactly 1 arg needed"
  exit 1
fi

# case
case $var in
 "$cond1") 
   commands
   ;;
 "$cond2") 
   commands
   ;;
 "$cond3") 
   commands
   ;; 
   *)
   commands
   ;;
 *)
 	 commands
 	 ;;

Loops

# reading a file line by line

while read line
do 
  commands
done < $"Filename"
# or
cat $"Filename" |
while read line
do 
  commands
done


for((i=0; i -lt 10; i++)); 

Examples

#!/bin/bash

while [ $# -gt 0 ]
do
  case $1 in
    -f|--file)
      FILE=$2
      if ! [ -f $FILE ]; then
        echo "This is not a valid file"
      exit 1
      fi
      echo "Words in the file $(cat $FILE | wc -w)"
      ;;
     -h|--help)
      echo "specifiy a file"
      ;;
  esac
done


while getopts a🅱️cd param; do
  case $param in 
    a)
     echo $OPTARG 1
     ;;
    b) 
     echo $OPTARG 2
     ;;
     
opts=`getopt -o -a:🅱️cd --long file::,name;,help -- "$@"`
eval set -- "$opts"

FUC

(Frequently Used Commands)

uname -a # extended system information.

tar cvf archive.tar folder/ # create archive.
tar xvf archive.tar # extract
tar tvf archive.tar  # view tar file.

find iname filename
find iname filename exec md5sum {} \; # find all files named filename and execute md5sum against them.
find ~ -empty # find all empty files.

ls -ltr # order files based on last modified time (reverse order)

gzip test.txt # create gz archive
gzip -d test.gz # unzip

unzip test.zip
unzip -l test.zip # view the contents of test.zip

shutdown -h now, reboot, init 6. Now they are the same, but used to be different internal process.
shutdown -h +10 # shutdown in 10 minutes.
shutdown -r now # reboot
shutdown -Fr now # check the filesystem on reboot.

IFS=$'\n' #new line separator

SED

For macOS use gsed, run brew install gnu-sed first to install it. Otherwise, some commands won’t work.

sed 's/wrod/word/g' <file.txt>
sed -n -e '1~2 p' test.txt # print every 2nd line
sed '4d' test.txt # delete the 4th line
sed '/The/$ p' test.txt # print file contents starting from the word "The" and to the end of the file ($)
sed '4 a hello world' test.txt # append "hello world" after the 4th line
sed '1~2 w test2.txt' test.txt # write every second line to test2.txt
sed '3,4 c hello world2' test.txt # replace the 3rd and the 4th lines with "hello world2"
sed '4 i hello world' test.txt # insert "hello world" before the 4th line of test.txt

sed '3 i hello\
world' test.txt # insert "hello" and "world" after the 3rd line, each word on the separate line

sed -n 'l 25' test.txt # line wrapping, after the 25th char
sed '3 q' test.txt # read the first 3 lines and quit
gsed -n '3p'  test.txt # print the 3rd line from the file
sed '3, 5 r junk.txt' test.txt # after lines 3 and 5 in test.txt instert the contents of the junk.txt
sed '/Paulo/ e date' books.txt # after the line containing "Paulo" insert current date
sed 'e' commands.txt # if 'e' command is empty and the "commands.txt" contains commands, they will be executed
sed 's/[[:digit:]]*$/Pages = &/' books.txt 

# sed replacements template: [address1[,address2]]s/pattern/replacement/[flags]
sed '/The Pilgrimage/ s/,/ | /g' books.txt # replace only when the line contains Pilgrimage
sed 's/,/ | /2' # replace the second occurence of comma only
gsed -n '/Two/s/,/ | /p' test.txt # show only changed lines (p at the end) 
sed -n 's/Paulo Coelho/PAULO COELHO/w junk.txt' books.txt # store changed lines in junk.txt


gsed -n '/#/p' test.txt # print all commented lines (for Python scripts and other languages that use # as a single line comment)

# with regex
echo "Three,One,Two" | sed 's|\(\w\+\),\(\w\+\),\(\w\+\)|\2,\3,\1|'

With sed not only / can be a delemeter. | , ^ , ! and @ can be used as well. This is handy, for example, when performing operations on the path, containing /.

https://www.tutorialspoint.com/sed/sed_managing_patterns.htm

https://www.udemy.com/course/linux-bash-shell-scripting-complete-guide-incl-awk-sed/learn/lecture/10129236#overview

On macOS sed is a BSD version with some weird and annoying behaviour. Install and use gsed instead. https://ben.lobaugh.net/blog/205337/how-to-fix-the-sed-command-expects-error-on-mac-os

# sed OPTIONS...SCRIPT[addrXoptions] INPUTFILE
sed '30,35d' table.txt # no OPTIONS, just SCRIPT and INPUTFILE. '30,35' - addresss (addr), d - option (delete). So, this script deletes lines 30,35 from INPUTFILE

# commands
sed '30a hello' table.txt # a - append "hello" text after the line
sed '30d' table.txt # d - delete text the line
sed '30i hello' table.txt # i text - insert "hello" text before line
cat table.txt | sed '/Peter/p' table.txt # p - print the pattern space. This command will print the line with Peter + all the lines including the one with Peter. So, Peter line will be orinted twice. Use -n to avoid this duplication: cat table.txt | sed -n '/Peter/p' table.txt
cat table.txt | sed -n '3p' table.txt # print the third line
cat table.txt | sed -n '3,$p' table.txt # print all lines starting from the third one

cat table.txt | sed -n '/HR/c Information hidden' table.txt # for lines with HR string change for Information hidden

# q[exit-code] - exit without further processing
# s/regexp/replacement/flags - substitute

sed -n ... # print the output only when -p is specified
sed -e ... # add script, meaning add another command
sed -e '/Mark/a after Mark' -e '/Mark/i before Mark' table.txt
sed -r ... # use extended regexp

sed -i ... # change the files
sed -i '/Peter/a after Peter' table.txt

sed '/Name/e echo -n "Date:"; date; echo "Output of pwd"; pwd' table.txt

sed 's/HR/& & &/' table.txt # for each line with HR in it, print HR three times
sed 's/[3-9][[:digit:]]/***/' table.txt # each line that contains a two digit number with the first digit in range between 3 and 9 and the second one being any number, and substitute for *** 

AWK

https://www.tutorialspoint.com/awk/awk_workflow.htm

https://www.udemy.com/course/linux-bash-shell-scripting-complete-guide-incl-awk-sed/learn/lecture/10129236#overview

awk 'BEGIN{print "Hello"}' # print Hello once
awk '{print "Hello"}' # print Hello each time Enter is pressed
awk '{print "Hello"}' filewithlines.txt  # print Hello for each line in the file
awk 'BEGIN{print "This line is printed once at the beginning"}{print "This is going to be printed one per line"}END{print "This is printed once at the end"}'

echo "one two three four" | awk '{print $0}' # print all fields: one two three four
echo "one two three four" | awk '{print $1}' # print the first field: one

cat table.txt | awk '{print "This is " $1, "from " $3 " department"}'
cat table.txt | awk '/Peter/ {print $1 " ", $3}' # only print info for Peters
cat table.txt | awk '!/Name/ {print $1 " ", $3}' # Do not print the header (only the header contains "Name")
cat table.txt | awk '$1 == "Peter"'

cat table.txt | awk 'END{print NF}' # print the number of fields
cat table.txt | awk 'END{print NR}' # print the number of records

cat /etc/passwd | awk 'BEGIN{FS=":"}{print $1 " " $7}' # change the separator for : and print the first and the seventh column
# same but shorter:
cat /etc/passwd | awk -F ":" '{print $1 " " $7}'
cat /etc/passwd | awk 'BEGIN{RS=":"}{print $1 " " $7}' # change the record separator (instead of newline)

# mathematical and boolean operators are the same as for C
cat /etc/passwd | awk 'BEGIN{RS="\n"; count=0}{count++}END{print NR, count}'

# if and loops also have the same syntax as C
cat table.txt | awk '{if ($1 == "Peter") {print $0}}'
cat /etc/passwd  | awk -F ":" '{if ($1 == "root") {print $1 " " $7}}' # print data for root

cat table.txt | awk '{for(i=1;i<=NF;i++){print "field:", i, $i}}'

System

Privileges

su - <user_name> # switch to user_name user
su - <user_name> -c 'ls' #  run ls command as user_name user

chmod 4777 prog # 4 - set-UID for the program enabled

cp /bin/sh /tmp/mysh && chmod 4755 /tmp/mysh # put a set-uid bit on the copied version. Now /tmp/mysh is run with the original user's privileges.

List Drives

sudo fdisk -l

USB

lsusb -v # linux
system_profiler SPUSBDataType # macOS, but not that verbose as lsusb on Linux
ioreg -p IOUSB # macOS, but not that verbose as lsusb on Linux
ioreg -p IOUSB -w0 -l # macOS, more details

Mounting

USB devices on Linux are mounted automatically, but internal drives need to be mounted manually. Also, on Linux you can mount a img file as a normal drive.

Remote

FTP

ftp <IP/hostname>
> mget *.html # get files
> mls *.html # view files

Processes

ps -ef | more # view currently running processes
ps -Hef | more # same but in a tree structure

top # top processes by CPU used. Type Shift+O to sort by columns
top -u <usename> # filter by user

kill 9 <PID> # terminate a process

Memory

free # show slack, free, used space
free -g # show RAM capabilties

df # system disk usage
df -h # in human-readable form (Gb)
df -T # file system type

Files

rm -i <filename> # remove with confirmation
rm -r <partial_or_full_name> . # recursive starting from the current directory

rmdir # rm dir if it's empty

cp -p <filename1> <filename2> # copy preserving attributes
cp -i <filename1> <filename2> # confirm if file exists

mv -i <filename1> <filename2> # rename filename1 to filename2 and ask to confirm
mv -f <filename1> <filename2> # overwrite without promting
mv -v <filename1> <filename2> # print what's happening during rename

cat <filename1> # show the file contents (text file only)
cat <filename1> <filename2> # concat filename1 and filename2

mkdir -p dir1/dir2/dir3 # create nested directories if don't exist.

locate <filename> # you will need to create index first, but that's done automatically when first run

tail -n <number_of_lines> <filename> # view last number_of_lines lines of a filename
tail -f <log_file> # view file in real time, useful for logs that are constantly changing

Information

files -e conf --stats # get stats for the files in the directory, having conf extension

Compare

cmp [-i N|-c|-l|-s]# test and bin
-i # ignore the first N symbols 
-c # output all differences
-l # return all differences
-s # return 0 if files are identical, 1 - differ, 2 - error encountered

diff # text only???
comm # used after sort. Returns three columns

⁉️ Where is the index for locate locates? Could be useful for 🔍 forensics?

Network

ifconfig -a # all interfaces + status
ifconfig eth0 up
ifconfig eth0 down

About

man- extensive. Better pipe to more or less.

whatis - signle line.

References

[1] Learn SED, Tutorials Point