- /cat/proc
- CPU info.
cat /proc/cpuinfo
cat /proc/timer_list - Thread info
cat /proc/sys/kernel/threads-max
# number of threads = total virtual memory / (stack size*1024*1024) - List RAID arrays
cat /proc/mdstat - List SCSI
cat /proc/scsi/scsi - List IDE
cat /proc/ide - List System Statistics
cat /proc/stat - List Kernel Variables
cat /proc/sys/ - List System Configuration Variables
getconf -a - System Statistics
procinfo - List IDE
cat /proc/ide
- CPU info.
- Files
- Compare two files
diff File_1.txt File_2.txt - Unrar files
unrar x Filename.rar - To compress, split and store the smaller files of size 650MB:
tar -cvj /home/username/movie/large-files.avi | split -b 650m -d – “large-files.tar.bz.” - To recover and extract the split files:
cat large-files.tar.bz.* > large-files.tar.bz - Extract files:
tar -xvj large-file.tar.bztar jxf /usr/src/linux-source-3.13.0.tar.bz2 - To Unzip multi-part zip files, first concatenate them together:
cat *.zip > test.zip - then unzip using the unzip program:
unzip -FF test.zip - Truncate File:
> FileName
truncate -s 0 filename
filename < /dev/null - Tee Terminal Output to File
./email-server.sh | tee out.txt
- Strip first 38 characters in file names
for file in *; do mv $file $(echo $file | sed -e 's/^....//'); done - Convert all files in the current directory from a file extension of .htm to .html.
# The -n means that it's a test run and will not actually change any files.
rename 's/\:/-/g' *.png -vn
rename -v ’s/\.htm$/\.html/’ *.htm
- Rename File to Uppercase
ls | while read upName; do loName=`echo "${upName}" | tr '[:upper:]' '[:lower:]'`; mv "$upName" "$loName"; done - # Replace Characters in Filenames:
rename "s/ /-/g" * - Strip the extension.JPG, that's what this piece of code does {i/.JPG} and then add new file extension
for i in *.JPG; do mv "$i" "${i/.JPG}".jpg; done - Use Basename ot strip extension and then add another
for i in *.jpg; do mv "$i" "`basename $i .jpg`.JPG"; done - Renaming the directories and files recursively
while IFS= read -r -d '' file; do
mv -b -- "$file" "${file,,}"
done < <(find path/to/dir -depth -name '*[A-Z]*' -print0) - Rename File to Uppercase
#!/usr/bin/env bash
cd pat/to/dir
for file in *; do
if [[ "$file" != "${file,,}" ]]; then
mv -b -- "$file" "${file,,}"
fi
done - Remove a part of the file name by using regular expression
rename -n 's/REGULAREXPRESSION//' *.html - delete all files in a directory except some
rm !(u|p)rm -R -- !(install.sh) - REMOVE directory
rm -rf /path/to/directory - Empty the trash folder
rm -rf ~/.local/share/Trash/* - This requires the following option to be set:
shopt -s extglob - du
- list size of all sub directories
du -ch - SIZE OF FOLDER
du -h | tail -n1 - Size of folders by size
du --max-depth=2 /home/ | sort -n -r - list size of all sub directories
du -ch - Disk usage summary for each partition
df -h
- list size of all sub directories
- ls
- Send output of command to screen and file:
ls -al | tee file.txt - List files in numerical order
ls -1v - Store ls results in an array
array=($(ls -1v */)) - list size of all sub directories
du -ch
- Send output of command to screen and file:
- Compare two files
- SYSTEM
- Manage Ubuntu startup programs, Configure Linux Startup Applications with
sysv-rc-conf - List partion table for disk devices
sfdisk -l - GET UBUNTU VERSION NAME
lsb_release -a - Get Bios Version
sudo dmidecode -s bios-version && sudo dmidecode -s bios-release-date
sudo dmidecode -t baseboard - CPU info.
crontab -e
*/10 * * * * /usr/bin/somedirectory/somecommand
# CRONTAB sends mail to /var/mail after completing a task. - Get Hostname
hostname -f - FIND COMMAND
find /usr -name nginx - Find location of binary or command
which php - Find details of binary
file /bin/ls - List command history
history - re-execute the most recent command
!! - re-execute the [number] command from your history list
!number - re-execute the most recent command with the pre-fix [prefix]
![prefix]
# (ex. !/usr/local/nginx/ will re-execute my most recent /usr/local/nginx/sbin/nginx run) - List UUID
sudo blkid - List IDE
cat /proc/ide - Install lm-sensors:
sudo apt-get install lm-sensors
# Then run:
sensors-detect - Sensors
apt-get install acpi
# Display all sensors:
acpi -V - Drop to command prompt and back
# Instead of Ctrl Alt F2, you can use Ctrl+Alt+Backspace to restart X
# return to your X session (Ctrl+Alt+F7)
- Manage Ubuntu startup programs, Configure Linux Startup Applications with
- System Hardware Information
- List all hardware
lshw - Get PCI devices
lspci - Get USB devices
lsusb - List all the hardware
hwinfo - prints the name, version and other details about the current machine and the operating system
uname or uname -a - Prints information about your CPU(s)
lscpu - Get List of All Installed Harddrives
sudo lshw -c disk - list installed hardware
lsdev
- List all hardware
- Linux Process
- See which services are running
sudo netstat -plunt | grep ssh - Get PID of process dd
ps -A | grep dd - Kill process
sudo kill -USR1 (PID)
sudo killall appname - Find process:
ps aux | egrep '(apache|httpd)' - prints the name, version and other details about the current machine and the operating system
uname - Watch Process
mkfifo /tmp/testpipetar cvf - / | gzip > /tmp/testpipe- Now open a new console and run this :
watch -n 1 'ps u -P $(pidof tar) - Open a third console and type :
cat /tmp/testpipe > /dev/null
# look at the watch cmd (2nd term) it will show an increase in cpu consumption !
- top
- Show Process'
top - real-time update of running system: memory use, processes
top -H
- Show Process'
- List all processes
ps -A - List processes by memory usage
ps -eo pmem,pcpu,rss,vsize,args | sort -k 1 -r | less - System load
cat /proc/loadavg - Load info
uptime - Realtime load info
xload
- See which services are running
- Filesystem
- LVM
- Install LVM
apt-get install lvm2 modprobe dm-modvgchange -a ylvm help
- Install LVM
- LVM
- SED
- SED - recursively replace this string in all files in a folder: style="color:orange"
find PATH-TO-FOLDER -type f -print0 | xargs -0 sed -i 's/style=\"color\:orange\"//g' - SED - recursively replace this string in all files in a folder:<h4 class="slideShow">(.)*</h4> with what is matched in the brackets ()
find PATH-TO-FOLDER -type f -print0 | xargs -0 sed -i 's/style=\"color\:orange\"//g' - SED - replace &
find . -type f -exec sed -i "s/\&/\&/" {} \; - SED - Print lines between articleText column2
sed -n -e '/articleText/,/column2/ p' fileName.html
find . -type f -exec sed -n -e '/articleText/,/column2/ p' {} \; - List files in numerical order and then SED them..then output to INDEX.html
i=`ls -1v $1`
for x in $i; do sed -n -e '/articleText/,/column2/ p' $x; done;
- SED - recursively replace this string in all files in a folder: style="color:orange"
- Replacing Strings
- REPLACE STRINGS
- AWK SED REPALCE STRINGS IN FILES
find . -type f -print0 | xargs -0 sed -i 's/subdomainA.example.com/subdomainB.example.com/g' - SED Replace in place
sudo sed -i "s/myhostname =.*/myhostname = ${HOSTNAMES[0]}/" /etc/postfix/main.cf
- AWK SED REPALCE STRINGS IN FILES
- NETWORKING
- Find Services Running on Port
netstat -atlpvnsudo lsof -i -n -Psudo netstat -tulnp
- Get IP address
ifconfig eth0 | grep inet | awk '{ print $2 }' - Arp manipulates or displays the kernel's IPv4 network neighbour cache.
arp -an - list size of all sub directories
du -ch - NETWORKING SPEED TEST
netperf -H 10.4.12.1
- Find Services Running on Port
- Firewall
- ufw
- Disable ufw
sudo ufw disable - or add ports to it via:
sudo ufw allow 8080 - allow TCP at 443
sudo ufw allow 443/tcp
- Disable ufw
- See Firewall rules:
sudo iptables -L - See Firewall rules:
sudo iptables -L -n - Open/Close port 8289:
sudo iptables -A INPUT -p tcp -d 0/0 -s 0/0 --dport 8289 -j ACCEPT - Check open ports
nmap localhost - See Open/Close ports:
sudo netstat -tulpn - LIST ALL PORTS
cat /etc/services - Commands to verify ports:
nmap IP#
nmap localhost
netstat –ntulp
- to verify single port
netstat -nap | grep port # - to list all current rules in iptables
iptables -L - For opening a TCP port:
iptables -A INPUT -p tcp –dport port# -j ACCEPT - For opening a UDP port:
iptables -A INPUT -p udp –sport port# -j ACCEPT - Save changes:
iptables-save > /etc/iptables.rules - If you need to disable the firewall temporarily, you can flush all the rules using:
iptables -F
- ufw
- wget
- Download entire website and change links to local links
wget \
--recursive \
--no-clobber \
--page-requisites \
--html-extension \
--convert-links \
--restrict-file-names=unix \
--domains marxists.org \
--no-parent \
https://www.marxists.org/history/ussr/sounds/
- Use Passwords to download page
wget -O - --http-user=username --http-passwd=password 'http://dynamic.zoneedit.com/auth/dynamic.html?host=www.mydomain.com' - Copy and Overwrite a File
wget http://business.gwu.edu/gwsbContent.css -O /var/www/gwchina.org/sites/all/themes/responsive_bartik/gwsbContent.css - Download directory contents
wget -r -np -R "index.html*" https://people.cs.clemson.edu/~westall/853/code/
- Download entire website and change links to local links
- wireshark
- Install wireshark
sudo apt-get install wireshark
sudo groupadd wireshark
sudo usermod -a -G wireshark YOUR_USER_NAME
sudo chgrp wireshark /usr/bin/dumpcap
sudo chmod 750 /usr/bin/dumpcap
sudo setcap cap_net_raw,cap_net_admin=eip /usr/bin/dumpcap
sudo getcap /usr/bin/dumpcap
- Install wireshark
- Memory
- Check free memory
free or free -m - Memory Usage
cat /proc/meminfo - Kernel caches
cat /proc/slabinfo - Swap partitions
cat /proc/swaps - Devices
cat /proc/devices - Interrupts
cat /proc/interrupts - Swap partitions
cat /proc/iomem - IO Ports
cat /proc/ioports - real-time update of running system: memory use, processes
top or top -H - Memory Usage
htop - Memory Usage
vmstat or vmstat -s - System page size
getconf PAGESIZE - List processes by memory usage
ps -eo pmem,pcpu,rss,vsize,args | sort -k 1 -r | less - Get RAM info
sudo dmidecode -t 17 - Free Memory on system
watch -n 5 free -m
- Check free memory
- PDF
- Combine PDFs
gs -dBATCH -dNOPAUSE -q -sDEVICE=pdfwrite -sOutputFile=finished.pdf file1.pdf file2.pdf - To merge two pdf files, file1.pdf and file2.pdf:
pdftk file1.pdf file2.pdf cat output mergedfile.pdf - GREP a PDF file.
sudo apt-get install python-pdfminer
pdf2txt myfile.pdf | grep mypattern - Convert PDF to image (png, jpg)
convert -density 150 Confirmation.pdf -quality 100 'VSC FollowUp.jpg'
- Convert to PNG
pdftoppm -png file.pdf prefix - Convert images to PDF
convert *.png Mathemtics.pdf - Append all images in a folder, vertically
convert -append *.jpg out.jpg - Increase the resolution (for higher quality output) as follows:
pdftoppm -rx 300 -ry 300 -png file.pdf prefix - To print only one page, use
pdftoppm -f N -singlefile -png file.pdf prefix
- Convert to PNG
- Combine PDFs
- Books
- Convert .epub to .mobi
ebook-convert "book.epub" "book.mobi" - Convert to PDF
man man | col -b > man.txt - Convert Man Pages to PDF
man -t man | ps2pdf – > man.pdf - Convert Man Pages to PDF
ebook-convert file.pdf file.epub --enable-heuristics
- Convert .epub to .mobi
- MAN Pages
- Convert Man Pages to HTML
man2html -title 'manpage_title' ascii_output_file html_file - Convert Man Pages to TXT
man man | col -b > man.txt - Convert Man Pages to PDF
man -t man | ps2pdf – > man.pdf - Convert PDF Pages to txt
pdftotext -layout input.pdf output.txt - Convert TEXT Pages to HTML
txt2html o.txt > a.html
- Convert Man Pages to HTML
- RAID
- Activate RAID on Ubuntu:
sudo vgscan - Fix the GRUB boot loader on your RAID drive(s).
-
To fix a non-bootable system and ensure GRUB is installed on both drives, manually install GRUB as follows:
mount /dev/md0 /mnt
chroot /mnt
grub
device (hd0) /dev/sda
root (hd0,0)
setup (hd0)
device (hd1) /dev/sdb
root (hd1,0)
setup (hd1)
quit
* Reboot the machine from the original Linux CDROM
* From the Linux boot menu, select “Rescue a broken system”
* Continue through the prompts until the screen “Device to use as a root file system” appears
* Press Alt-F2 to switch to a second console screen then press Enter to activate it.
* Mount the md0 RAID device and use chroot and grub to install the bootloader onto both sda and sdb using the following commands - Ubuntu System on Software RAID
- Install grub to the hard drive
grub-install --modules=raid --no-floppy /dev/sdb
apt-get install mdadm
update-initramfs -u
update-initramfs -u -ck all
mdadm --assemble --scan
- Install grub to the hard drive
- Activate RAID on Ubuntu:
- Package Manager
- # As a GENERAL fix issue the following commands ONE AT A TIME:
sudo dpkg --configure -a
sudo apt-get -f install
sudo apt-get --fix-missing install
sudo apt-get update
sudo apt-get dist-upgrade
sudo apt-get clean
sudo apt-get autoremove
sudo apt-get update
- # To manually remove the package
cd /var/lib/dpkg/info
sudo rm flashplugin-nonfree.*
#Now remove the offending packages:
sudo dpkg --remove --force-remove-reinstreq flashplugin-nonfree
sudo dpkg --remove --force-remove-reinstreq realplayer
- # List all installed software
apt --installed list
- # Unmet Dependencies APT-GET UPDATE FORCE
sudo apt-get clean
sudo apt-get autoclean
sudo apt-get -f install
sudo dpkg --configure -a
sudo apt-get -u dist-upgrade
sudo apt-get -o Debug::pkgProblemResolver=yes dist-upgrade
sudo apt-get remove --dry-run package-name
- # LIST WHERE ALL A PROGRAM IS INSTALLED
sudo dpkg -L apache2
- # Search for packages:
apt-cache search keyword
- c++
- # As a GENERAL fix issue the following commands ONE AT A TIME:
- Run C++ program x number of times
for i in {1..40}; do ./a.out >> out.txt; done - Gather output from all runs of the C++ program
for i in {1..1000}; do ./a.out; done | sort | uniq -c - Post Preprocessor Stage, C++ Compiler
g++ -std=c++11 -E main.cpp > main.ii - Use threads and c++11 features of c++:
g++ timer.cpp -std=c++11 -lpthread - Ask gcc where it searched for includes
`gcc -print-prog-name=cc1plus` -v
# In addition, gcc will look in the directories specified after the -I option. - Search C++ man pages
man std::iostream
man __gnu_cxx::slist
man namespace::header - Make - lots of useful debugging info regarding what went wrong:
make - d - List shared object dependencies on an a.out or .so files files (i.e. which shared objects need to be loaded at runtime to run the a.out or with loading the .so):
ldd a.out
- Readelf - investigate an object file:
readelf --symbols ./hello.o - List dynamic symbol table entries from a .so file (e.g write):
objdump -T /lib/libc.so.6 | grep write - View binary file - investigate an object file:
hexdump -C a.out| more - On an Ubuntu System place libraries here:
/usr/local/lib - Permanently set LD_LIBRARY_PATH to search for C++ libraries (in Ubuntu). As "root", do:
echo '/home/yash/Documents/CODE/C++/libs' >> /etc/ld.so.conf.d/cpplibs.conf
- Run C++ program x number of times
- Hard Drives
- Clear MBR
sudo dd if=/dev/zero of=/dev/sda bs=512 count=1 - Check free space on harddrive
df -h - Get Harddrive Info
smartctl -a /dev/sda
sudo smartctl --all /dev/sdb - Hard Disk info.
sudo hdparm /dev/sda -
# To see the contents of your MBR MASTER BOOT RECORD on harddrive.
# The dd command, which needs to be run from root, reads the first 512 bytes
# from /dev/hda and writes them to the mbr.bin file. The od command prints
# the binary file in hex and ASCII formats.
dd if=/dev/hda of=mbr.bin bs=512 count=1
od -xa mbr.bin
smartctl --test=short /dev/sda
- Clear MBR
- Users and Groups
- List user's uid, gid, groups, etc.
id
who
whoamI - Encrypt User's home directory
sudo adduser --encrypt-home yash - List all Groups
cat /etc/group - Delete user from group
deluser user group - Add user from group
sudo usermod -a -G GROUP USER - List groups a user belongs to
groups username - List all users:
cut -d: -f1 /etc/passwd - Add user:
sudo adduser new_username
#When creating new user open /etc/passwd file and change the shell /bin/sh to /bin/bash
- To remove/delete a user:
sudo userdel username
# delete the home directory for the deleted user account :
sudo rm -r /home/username - Modify the username of a user:
usermod -l new_username old_username - Change the password for a user:
sudo passwd username - Change the shell for a user:
sudo chsh username - Change the details for a user:
sudo chfn username
- List user's uid, gid, groups, etc.
- FTP
- SFTP
- Copy remote directory recursively
get -r someDirectory - Maintain the appropriate permissions and access times
get -Pr someDirectory
- Copy remote directory recursively
- LFTP
- Download in 5 parts
lftp -e 'pget -n 5 http://kernel.org/pub/linux/kernel/v2.6/linux-2.6.22.2.tar.bz2' - Backup entire site
lftp -u username,password -e "set ftp:ssl-allow no; mirror -c www/ /var/www; quit" ftp.mysite.com - Use 10 in parallel
mirror -c --parallel=10 - LFTP Use lftp as download accelerator
lftp -e 'pget -n 5 ftp://path/to/file'
- Download in 5 parts
- RSYNC
- Synchronize two folders
rsync -avv SourceFolder DestinatioDrive - Maintain the appropriate permissions and access times
get -Pr someDirectory
- Synchronize two folders
- SFTP
- FIND
- FIND COMMAND
find /usr -name nginx
- Grep Recursive search
grep -r "texthere" .
grep -r --include "*.txt" texthere .
# exclude Files with --exclude. - Grep output only matched elements
grep -oh id\"\:[0-9]* latestTrades.txt - FIND COMMAND
find /usr -name nginx - List UUID
sudo blkid - List SCSI
cat /proc/scsi/scsi - List IDE
cat /proc/ide - List System Statistics
cat /proc/stat - List Kernel Variables
cat /proc/sys/ - List System Configuration Variables
getconf -a - System Statistics
procinfo - List IDE
cat /proc/ide
- FIND COMMAND
- GDB
- GDB handles abreviations of commands
help topicName - List the stack (sequence of function calls) executed
where (or bt) - instruction location and source code line where program was aborted
where
frame frameNumber
- Stack frames
frame
frame frameNumber
- Debug executable with GDB
gdb ./seg_fault
- Breakpoints
- Starts program execution from the beginning
run - Set break point
break lineNumber - Set breakpoint at begining of function
break FuncName - Continue execution from breakpoint
continue - Set breakpoint breakPointNumber where expresison is true
condition breakPointNumber expression - List all breakpoints
info break - Disable breakpoints
disable breakpoints - Enable breakpoints
enable breakpoints - Clear breakpoint at lineNumber
clear lineNumber - Deletes breakpoint number breakPointNumber
delete breakPointNumber - Delete all breakpoints
delete
- Step
- Executes next line of program
step (or s) - Executes next numberOfLines of program
step numberOfLines - Treat a function call as a single step
next (or n) - Executes program until lineNumber
until lineNumber
- Display Information
- List information about current stack frame
info frame - List local variable values of current stack frame
info locals - List argument values of current stack frame
info args - List register values
info registers - Displays the value of expression
print expression (or inspect expression) - Each time a breakpoint is hit, display this expression
display expression - List data type of expression
whatis expression - IMPORTANT - List local variables in current stack frame
info locals - IMPORTANT - Set variable value
set variable variableName = expression
- List
- Lists next few lines of program
list - List lines around line number lineNumber
list lineNumber - List line numbers START through STOP
list START STOP - List lines at begining of functionName
list functionName - IMPORTANT - List local variables in current stack frame
set variable variableName = expression
- Quit gdb
quit
- GDB handles abreviations of commands
- Strace
- Strace: Inspect system calls performed by program
find /usr -name nginx
- Strace: Filtering:
strace -e - Strace: Timing:
strace -t[tt] / strace -T - Strace: Statistics:
strace -c - Trace subprocesses:
strace –f ./a.out - Save strace output to a file, trace all system calls
strace –o strace.out ./a.out - Limit tracing to particular group of system calls
strace –e trace=network ./a.out - Enable verbose output
strace –v ./a.out - Limit tracing to a particular group of system calls
strace –e trace=open,execve ./a.out - Include timing information in output
strace –tt ./a.out - Attach to a running process
strace –p PID - Strace: Timing:
strace -t[tt] / strace -T - Strace: Timing:
strace -o /output/file -f -tt NAMEOFCOMMAND
- Strace: Inspect system calls performed by program
- Valgrind
- VALGRIND: Enable leak checking (shows line numbers)
valgrind --leak-check=full ./a.out - VALGRIND: Track leak origins
valgrind --leak-check=full --track-origins=yes ./a.out - VALGRIND: Attach to a debugger on certain errors
valgrind --db-attach=yes --leak-check=full --track-origins=yes ./a.out
- VALGRIND: Enable leak checking (shows line numbers)
- ltrace
- ltrace: trace library calls
ltrace -o librarycalls.txt ./weirdpiping - ltrace: Filtering:
ltrace -e - Strace: Timing:
ltrace -t[tt] / -T - Strace: Statistics:
ltrace -c
- ltrace: trace library calls
- HOPS
- To determine the number of hops, specify the TTL (Time-to-Live) in the ping command using the ‘-t’ option. Increase the TTL value till ping does not return any ICMP related error. The smallest TTL value with no errors, is the number of hops required to reach the destination.
ping -c 2 -t15 99.84.219.185
- To determine the number of hops, specify the TTL (Time-to-Live) in the ping command using the ‘-t’ option. Increase the TTL value till ping does not return any ICMP related error. The smallest TTL value with no errors, is the number of hops required to reach the destination.
- nmap
- Scan destination ports
nmap --script ssl-enum-ciphers ec2-34-239-168-74.compute-1.amazonaws.com
- Scan destination ports
- R
- MATH: R: Invoke R from command line and display summary statistics
R -q -e "x <- read.csv('out.txt', header = F); options(scipen=x); summary(x); sd(x[ , 1])" - MATH: R: Read file into data frame:
v=read.table("out.txt") - MATH: R: Calculate Std. Dev from column in data frame
sd(v[ , 1]) - MATH: R: Calculate Std. Dev from column in data frame
options(scipen=x) where x is the number of decimal places you want to display - MATH: R: View distribution of data points as a histogram
hist(v[ ,1], breaks=12, col="red") - MATH: R: View distribution of data points as a density plot
d <- density(v[ ,1])
plot(d, main="Kernel Density of Miles Per Gallon")
polygon(d, col="red", border="blue") - MATH: R: Just plot a vector in a data frame
plot(v[ ,1]) - MATH: R: Combine two data frames as two adjacent columns (assuming you have same number of rows)
v3 < - cbind(v2, v1) - MATH: R: Plot combined data frame
plot(v3) - MATH: R: Change column names for dataframe v3
colnames( v3 )[1] < - 'v1'
colnames( v3 )[2] < - 'v2' - MATH: R: Invoke R from command line and display summary statistics
- MATH: R: Turn off Scientific notation
options(scipen=999) - MATH: R: Save plot in variable myPlot
myPlot=ggplot(v3, aes(x=v1, y=v2)) + geom_point(shape=1, alpha=1/4) + geom_smooth(method="lm", col="firebrick") + coord_cartesian(xlim=c(1554631804906, 1554637306426), ylim=c(5107.01000000, 5149.00000000)) - MATH: R: Now export myPlot as SVG
ggsave(file="test.svg", plot=myPlot, width=10, height=8) - MATH: R: Plot Histogram (of one column)
ggplot(v3, aes(x=v2) ) + geom_histogram(color="black", alpha=.2, fill="#FF6666") + geom_density(alpha=.2, fill="#FF6666") + labs(x="BTC Trading Price (USD)", y="Number of Trades") + labs(title="Breakdown of Trades by Price")
ggplot(v1, aes(x=v3[,1]) ) + geom_histogram(bins=90, color="black", alpha=.2, fill="#336699") + labs(x="Time in Nanoseconds (Total: 1hr 30mins)", y="Number of Trades") + labs(title="Breakdown of Trades by Time") - MATH: R: Plot from Bash shell
R -q -e "library(ggplot2); \ v1=read.table('BTCTimestampData.txt'); \ v2=read.table('BTCPriceData.txt'); \ v3 <- cbind(v1,v2); \ h <- c('v1', 'v2'); \ colnames(v3) <- h; \ myPlot=ggplot(v3, aes(x=v1, y=v2)) + geom_point(shape=1, alpha=1/4) + labs(x='Time in Nanoseconds (Total: 12 hours)', y='BTC Price (USD)') + labs(title='BTCUSDT Trading Data') ; \ ggsave(file='btc.svg', plot=myPlot, width=50, height=10,limitsize = FALSE) ;"
- MATH: Calculate average (confirmed, this works)
awk '{s+=$1}END{print "ave:",s/NR}' RS="\n" out.txt - MATH: Calculate Standard Deviation (bash shell):
awk '
BEGIN {sum=0;summ=0}
{
arr[NR]=$1
sum+=$1
}
END {
average=sum/NR
for (i=1;i<=NR;i++) {
summ+=(arr[i]-average)^2
}
print sqrt(summ/(NR-1))
}
' data.txt - MATH: Calculate Variance
# Confirmed, this works
awk '
BEGIN {sum=0;summ=0}
{
arr[NR]=$1
sum+=$1
}
END {
average=sum/NR for (i=1;i<=NR;i++) { summ+=(arr[i]-average)^2 } print summ/(NR-1) } ' out.txt - MATH: Find max
sort -n myfile | head -n1 - MATH: Find min
sort -n myfile | tail -n1 - GNUPLOT:
gnuplot
plot "out.txt" with lines
- Install Virtualbox Extension Pack
sudo vboxmanage extpack uninstall "Oracle VM VirtualBox Extension Pack" - Install Virtualbox Extension Pack
sudo vboxmanage extpack install Oracle_VM_VirtualBox_Extension_Pack-4.3.18-96516.vbox-extpack
- Stop the MySQL Server.
sudo /etc/init.d/mysql stop - Start the mysqld configuration.
sudo mysqld --skip-grant-tables & - Login to MySQL as root.
mysql -u root mysql - Replace YOURNEWPASSWORD with your new password!
use mysql;
update user set password=PASSWORD('pwd') where User='root';
flush privileges; - Start the mysqld configuration.
sudo mysqld --skip-grant-tables & - Start the mysqld configuration.
sudo mysqld --skip-grant-tables &
- Track packets to and from 1.1.1.1
tcpdump host 1.1.1.1
- Get image info:
identify nameofimage.jpeg - Compress losslessly (default will overwrite existing image):
jpegoptim nameofimage.jpeg - Make Thumbnails:
mogrify -resize 200x200 -gravity center -extent 128x128 -format png -quality 75 -background transparent -path thumbs *.jpg - Make Sprite:
feh -m --sort filename -W 900 -H 900 -y 120 -E 120 *.png
feh -m -S name -W 900 -H 750 -y 120 -E 120 *.png
Extract audio stream from mkv files in a folder and move the output to another folder:
for i in ls ~/Videos/'Alan Watts - Wiara'/*.mkv ; do ffmpeg -y -i "$i" -map 0:a -c copy ~/Music/'Alan Watts - Wiara'/"$(basename "$i" .mkv).mkv" ; done;Extract audio stream from webm files in a folder and move the output to another folder:
for i in ls ~/Videos/'Alan Watts - Wiara'/*.webm ; do ffmpeg -y -i "$i" -map 0:a -c copy ~/Music/'Alan Watts - Wiara'/"$(basename "$i" .webm).webm" ; done;Extract audio stream from mp4 files in a folder and move the output to another folder:
for i in ls ~/Videos/'Alan Watts - Wiara'/*.mp4 ; do ffmpeg -i "$i" -vn -acodec copy ~/Music/'Alan Watts - Wiara'/"$(basename "$i" .mp4).aac" ; done;- Extract AAC audio from an MP4
ffmpeg -i NameOfVideo.mp4 -vn -acodec copy output_audio.aacffmpeg -i NameOfVideo.mp4 -vn -acodec copy output_audio.m4a - Extract audio from .mp4 video files in a folder and copy to a subfolder called out (create "out" in your sub-directory)
for i in *.mp4 ; do ffmpeg -i "$i" -vn -acodec copy out/"$(basename "$i" .mp4).aac" ; done; - Extract audio from a .mkv video file
ffmpeg -i YOURVIDEFILE.mkv -map 0:a -c copy output.mkv - Extract audio from .mkv video files in a folder and copy to a subfolder called out (create "out" in your sub-directory)
for i in *.mkv ; do ffmpeg -i "$i" -map 0:1 -map 0:a -c copy out/"$i" ; done; - Encode MP3 from a video file
ffmpeg -i nameofvideofile.mkv -map 0:a -vn -b:a 320k my.mp3 - Encode MP3s from a video files in a folder, note the *
for i in /PATH/T0/YOUR/FOLDER/* ; do ffmpeg -i $i -map 0:a -vn -b:a 320k mp3/$i.mp3 echo $i; done; - Extract audio between 5 and 10 seconds
ffmpeg -i input_video.avi -ss 5 -to 10 -vn -acodec mp3 output_audio.mp3 - Extract video between at 00:00:07 and 00:50:07
ffmpeg -ss 00:00:07 -i orginalfile -t 00:50:07 -vcodec copy -acodec copy newfile - Find the audio stream # in your video file using ffprobe
ffprobe nameofile.mkv - ALSA Ubuntu Audio
aplay -l - Speaker Test
aplay -D plughw:0,3 /usr/share/sounds/alsa/Front_Center.wav
# or
speaker-test -D plughw:0,3 -c 2
- Combine MP4 files
sudo apt-get install gpac -y
MP4Box -cat part1.mp4 -cat part2.mp4 -new joinedfile.mp4
DO NOT USE MP3s. Instead use .aac or .m4a
- To download an entire channel
youtube-dl -citw ytuser:youtubeUSERNAME - To download all videos on a playlist
youtube-dl -citw PLAYLIST-URL
- To download the entire channel
youtube-dl -f best -ciw -o "%(title)s.%(ext)s" -v url-of-channel
- List user's environment variables
env - Process exit status
ls -al file.ext
rc=$?
if [[ $rc != 0 ]] ; then
exit $rc
fi
- Interrupts
kill -l
- SSH Generate SSH keys
ssh-keygen -t rsa -b 4096
- Screen: To disconnect (but leave the session running)
Hit Ctrl + A and then Ctrl + D in immediate succession. You will see the message [detached]
- Go to end of line
$ - Go to end of line and enter Insert mode (aka Append mode)
A - Go to last non-blank character on line
g_ - Go to first non-blank character on line
^ or _ - Go to start of line including whitespace
0 - Go to start of line, switch to insert mode
0i or I - Go to end of document
G - Go to the begining of page
1G or gg - Copy line (press Esc first)
y - Paste line
p - Copy line including the return character at end of line
yy or Y - Cut line
d - Select certain characters
v - Select entire line
V - yank the current word (excluding surrounding whitespace)
yiw - yank the current word (including leading or trailing whitespace)
yaw - yank from the current cursor position up to and before the character (til x)
ytx - yank from the current cursor position up to and including the character (find x)
yfx - yank to the end of the current line (but don't yank the newline character)
y$
RAID Recover from bad superblock
First copy the drive from the corrupt drive to the new drive. The logfile will record any errors.
Copy sdb to sdc:
ddrescue -r3 /dev/sdb /dev/sdc logfile
sudo ddrescue -v --no-split /dev/sdc /dev/sdb logfile -f
Note: Your new drive should be equal to or greater than the size of the corrupt RAID array.
e2fsck -v -f /dev/sdc
mount -t ext2 -o ro /dev/sdc /mnt
If you know where an alternative superblock is (use dumpe2fs /dev/sdc | grep -i superblock to get alternative superblocks) then you can try mounting the rescued image using that block (sb):
mount sb=12345 /dev/sdc /mnt
The following command will scan the harddrive for superblocks. It might take upto 48 hours:
e2fsck -cc /dev/md127
Either the superblock or the partition table is likely to be corrupt!
Abort < y > ?
Answer n
# resize2fs /dev/md127
# fsck -f /dev/md127
To repair the filesystem using alternate superblock, run:/
fsck.ext4 -p -b superblock -B blocksize /dev/sdc
- - c is to enable 32-bit I/O support (x : 0 to disable, 1/2/3 to enable)
- - d indicates using dma (x : 1 to enable, 0 to disable)
- - u stand for interrupt-unmask flag (x :1 to enable, 0 to disable)
- - m is for multisector I/O (xx can be 2,4, 8 and 16 or any maximum number support by HDD, check using -i)
- - X set the transfer mode of hard drive.
Smartmontools: Hard Drive Health
Install Smartmontools
sudo apt-get install smartmontools
sudo apt-get install gsmartcontrol
Make sure SMART is enable on drive /dev/sda:
sudo smartctl -s on /dev/sda
Initiate test:
sudo smartctl -t long /dev/sda
View Results
sudo smartctl -l selftest /dev/sda
For IDE drives:
sudo smartctl -a /dev/sda
For SATA drives
sudo smartctl -a -d ata /dev/sda
mdadm
Create RAID 0 (level=0) array:
mdadm --create --verbose /dev/md0 --level=0 /dev/sda1 /dev/sdb2
If creating a RAID 1 array then you can image the first drive:
sfdisk -d /dev/sda | sfdisk /dev/sdb
Get RAID array information:
mdadm --detail /dev/md0
Delete RAID 0 array:
mdadm --stop /dev/md0
mdadm --remove /dev/md0
mdadm --zero-superblock /dev/sda
If you are not able to completely destroy the array, nuke the drive:
As an aside, you can extract the Master Boot Record of a drive using this:
dd if=/dev/sda of=mbr.bin bs=512 count=1
od -xa mbr.bin
Stop RAID arrays:
mdadm --stop /dev/md1
mdadm --stop /dev/md0
Start RAID arrays:
mdadm --assemble --scan
Start RAID arrays:
mdadm --detail --scan >> /etc/mdadm/mdadm.conf
Add disk (/dev/sdc1) to RAID array md0:
mdadm --add /dev/md0 /dev/sdb1
Create disk (/dev/sda1) from RAID array md0:
mdadm /dev/md0 --fail /dev/sda1 --remove /dev/sda1
hdparm
Test hard drive read speed:
sudo hdparm -t /dev/sda
Test hard drive read speed by skipping the first 500 GB of data:
hdparm -t --direct --offset 500 /dev/sda
Get hard drive information:
sudo hdparm -I /dev/sda
Run Hdparm by directly reading data from the disk without using the buffer kept by the kernel to speed up data delivery:
hdparm -t --direct /dev/sda
All these settings will be forgotten on reboot. To make these changes permanent, edit /etc/hdparm.conf on Debian systems.
Tuning: Force drive to deliver data from say 20 sectors at once.
hdparm -m20 /dev/sda
Check: The number of sectors that can be deliverd by the hard drive.
hdparm -i /dev/sda
Tuning: Force drive to read 256 sectors in advance of the next read request
hdparm -a256 /dev/sda
Tuning: Force queries from the operating system to reach the hard drive controller faster
hdparm -c3 /dev/hda
Tuning: Enable Multiword DMA mode2
hdparm -X34 -d1 -u1 /dev/hda
Tuning: Enable write-back caching (the hard drive first stores the data to be written in a buffer before starting to write it)
hdparm -W1 /dev/sda
Tuning: All in one
hdparm -cx -dx -ux -mxx -Xxx /dev/hda
Measure hard drive temperature
sudo hdparm -H /dev/sdf