"""
.. note::
This project is under active development.
"""
import os
import glob
import sys
import astropy.io.fits
import matplotlib
import numpy as np
import time
import timeit
import skimage.transform
import psycopg2.extras
import psycopg2
from PIL import Image, ImageDraw, ImageFont
import pandas as pd
import pandas.io.sql as psql
from sqlalchemy import create_engine
import matplotlib.pyplot as plt
import re
from scipy import interpolate
from copy import copy
sys.path.append(os.path.join(os.path.dirname(__file__), "..", "radiospectra"))
module_path = os.path.abspath(os.path.join('radiospectra'))
if module_path not in sys.path:
sys.path.append(module_path)
[docs]def signal_to_noise(arr):
"""Calculate the signal-to-noise ratio of the input data.
:param array_like arr: an array_like object contain the data.
:returns: The signal-to-noise ratio of arr, here defined as the mean divided by the standard deviation.
:rtype: float
"""
#arr = np.asanyarray(arr)
m = arr.mean()
std = arr.std()
return m/std
[docs]def get_abs_data(arr):
"""Get the absolute values from the arrays.
:param float arr: the data in the arrays from the spectrograms.
:returns: Return an array with absolute values.
:rtype: float.
"""
abs_data = np.absolute(arr.data.flatten())
return abs_data
[docs]def get_min_data(data1, data2):
"""Get the minimum value from the both data1 and data2.
:param float data1: the data from spectrogram using the function 'Constbacksub + elimwrongchannels'
:param float data2: the data from spectrogram using the function 'subtract_bg_sliding_window'
:returns: Return the minimum values from data1, data2
:rtype: float.
"""
min_value = int(min(np.nanmin(data1), np.nanmin(data2)))
return min_value
[docs]def get_max_data(data1, data2):
"""Get the maximum value from the both data1 and data2.
:param float data1: the data from spectrogram using the function 'Constbacksub + elimwrongchannels'
:param float data2: the data from spectrogram using the function 'subtract_bg_sliding_window'
:returns: Return the maximum values from data1, data2
:rtype: float.
"""
max_value = int(max(np.nanmax(data1), np.nanmax(data2)))
return max_value
[docs]def move_axes(fig, ax_source, ax_target):
"""Move the axes to create the new Figure."""
old_fig = ax_source.figure
ax_source.remove()
ax_source.figure = fig
ax_source.set_ylabel('')
ax_source.set_xlabel('')
ax_source.set_position(ax_target.get_position())
ax_target.remove()
ax_target.set_aspect("equal")
fig.axes.append(ax_source)
fig.add_subplot(ax_source)
plt.close(old_fig)
[docs]def get_all_instruments(database, sql_query):
"""Get the all instruments data from the Database.
:param database: a database 'Validation'.
:param sql_query: Sql query from the database.
:returns index: index of the cursor frm database.
:returns cursor: the cursor frm database.
"""
sql_query_instruments = sql_query
cursor = database.cursor(cursor_factory=psycopg2.extras.DictCursor)
cursor.execute(sql_query_instruments)
index = [row for row in cursor.fetchall()]
return index, cursor
[docs]def plot_spec(rows):
"""Get the plots in four columns.
:param rows: the rows from the database.
:returns: return four columns (Original Data, ‘Constbacksub + elimwrongchannels’ , subtract_bg_sliding_window, Histograms).
:rtype: Matplotlip figure.
:exception: The buffer is too small for the requested array!
"""
database = psycopg2.connect(host= test_config.DB_HOST,
user= test_config.DB_USER,
database= test_config.DB_DATABASE,
port= test_config.DB_PORT,
password= test_config.DB_PASSWORD)
sql_query = "select * from validation_data"
rows = get_all_instruments(database, sql_query)
for row in rows:
try:
# First column Original Data
spec = CallistoSpectrogram.read(
test_config.DATA_PATH + row[1])
fig1, ax1 = plt.subplots(1, 4, figsize=(20, 5))
ax1 = spec.plot()
ax1.title.set_text("Original Data")
plt.close()
# Second column, Constbacksub + elimwrongchannels
spec2 = spec.subtract_bg(
"constbacksub", "elimwrongchannels")
fig2 = plt.subplots(1, 4, figsize=(20, 5))
ax2 = spec2.plot(vmin=-5, vmax=5)
ax2.title.set_text("Background subtracted")
plt.close()
# Third column, subtract_bg_sliding_window
spec3 = spec.subtract_bg("subtract_bg_sliding_window", window_width=800, affected_width=1,
amount=0.05, change_points=True)
fig3 = plt.figure(figsize=(20, 5))
ax3 = spec3.plot(vmin=-5, vmax=5)
ax3.title.set_text(
"Gliding background subtracted (window=800)")
plt.close()
# Fourth column, Histograms
data_absolute3 = get_abs_data(spec2)
data_absolute4 = get_abs_data(spec3)
fig4, ax4 = plt.subplots(figsize=(20, 5))
# take the min and max from the data to set the bins.
min_value = get_min_data(data_absolute3, data_absolute4)
max_value = get_max_data(data_absolute3, data_absolute4)
ax4.hist(data_absolute3, histtype='step', bins=range(min_value, max_value + 1),
label='Background subtracted')
ax4.hist(data_absolute4, histtype='step', bins=range(min_value, max_value + 1),
label='Gliding background subtracted')
# Calculate the standard deviation and signal-to-noise => rounded them to have 3 digits.
std_data = round(np.std(data_absolute4), 3)
snr_data = round(signal_to_noise(data_absolute4), 3)
# Set title for the histograms and show the std/snr values.
ax4.title.set_text(
f"Histograms, std = {std_data}, snr = {snr_data}")
plt.legend()
# Plot final plot by moving axes to the figure
fig_target, (axA, axB, axC, axD) = plt.subplots(
1, 4, figsize=(30, 7))
plt.suptitle(fig1._suptitle.get_text())
move_axes(fig_target, ax1, axA)
move_axes(fig_target, ax2, axB)
move_axes(fig_target, ax3, axC)
move_axes(fig_target, ax4, axD)
for ax in (ax1, ax2, ax3):
ax.set_xlabel('Time[UT]')
ax.set_ylabel('Frequency[MHz]')
ax4.set_xlabel('Pixel values')
ax4.set_ylabel('Number of pixels')
plt.show()
plt.close()
except Exception as err:
print("The Error message is: %s and the file name is %s" %
(err, row[2]))
[docs]def get_pdf_file():
"""Save the spectrograms into the pdf file.
:returns: return A pdf file containing the Plotted Spectrograms with four columns
:returns: pdf file.
"""
with PdfPages('BgSubImages_test.pdf') as pdf:
plot_spec()
print("Finished plotting!")
[docs]def update_all_values():
"""Calculate the std and snr, then update them into the table in Database.
:exception: The buffer is too small for the requested array!
"""
database = psycopg2.connect(host=test_config.DB_HOST,
user=test_config.DB_USER,
database=test_config.DB_DATABASE,
port=test_config.DB_PORT,
password=test_config.DB_PASSWORD)
rows, cursor = get_all_instruments(database)
for row in rows:
try:
spec = CallistoSpectrogram.read(test_config.DATA_PATH + row[1])
spec2 = spec.subtract_bg("subtract_bg_sliding_window", window_width=800, affected_width=1,
amount=0.05, change_points=True)
data = np.absolute(spec2.data.flatten())
std_data = np.std(data)
snr_data = signal_to_noise(data)
sql_update_query = f"""UPDATE data SET std = {std_data}, snr= {snr_data} where id = {row[0]} """
cursor.execute(sql_update_query)
database.commit()
except Exception as err:
print("The Error message is: %s and the file name is %s" %
(err, row[2]))
print("Update Done!")