
import pandas as pd

dataset = pd.read_csv(
    "https://edunet.kea.su/repo/EduNet-web_dependencies/datasets/airline-passengers.csv"
)
dataset.head()


import matplotlib.pyplot as plt

training_data = dataset.iloc[:, 1:2].values  # transform dataframe to numpy.array
# plotting
plt.figure(figsize=(12, 4))
plt.plot(training_data, label="Airline Passangers Data")
plt.title("Number of passengers per month")
plt.ylabel("#passengers")
plt.xlabel("Month")
labels_to_display = [i for i in range(training_data.shape[0]) if i % 12 == 0]
plt.xticks(labels_to_display, dataset["Month"][labels_to_display])
plt.grid()
plt.show()


import statsmodels.api as sm

test = sm.tsa.adfuller(training_data)
print("adf: ", test[0])
print("p-value: ", test[1])
print("Critical values: ", test[4])
if test[0] > test[4]["5%"]:
    print("The time series is not stationary")
else:
    print("The time series is stationary")

adf:  0.8153688792060528
p-value:  0.9918802434376411
Critical values:  {'1%': -3.4816817173418295, '5%': -2.8840418343195267, '10%': -2.578770059171598}
The time series is not stationary


# Calculate the trend
trend = dataset["Passengers"].rolling(window=12).mean()
trend.plot(figsize=(12, 2))
plt.xlabel("Month")
plt.ylabel("Trend")
plt.show()


# Calculate the seasonality
seasonality = dataset["Passengers"] - trend
seasonality.plot(figsize=(12, 2))
plt.xlabel("Month")
plt.ylabel("Seasonality")
plt.show()


test = sm.tsa.adfuller(seasonality.dropna())
print("adf: ", test[0])
print("p-value: ", test[1])
print("Critical values: ", test[4])
if test[0] > test[4]["5%"]:
    print("The time series is not stationary")
else:
    print("The time series is stationary")

adf:  -3.1649681299551475
p-value:  0.02210413947387869
Critical values:  {'1%': -3.4865346059036564, '5%': -2.8861509858476264, '10%': -2.579896092790057}
The time series is stationary


# first-order difference
dfOrigin = dataset["Passengers"]
df1 = dfOrigin.diff(1).dropna()
# second-order difference
df2 = df1.diff(1).dropna()
# third-order difference
df3 = df2.diff(1).dropna()

# plot three curves and check the stationary
fig, (ax1, ax2, ax3) = plt.subplots(nrows=3, ncols=1, sharex=True)

ax1.plot(df1)
ax1.set_title("first order")

ax2.plot(df2)
ax2.set_title("second order")

ax3.plot(df3)
ax3.set_title("third order")

plt.show()


def check_stationary(ts_data):
    df_test = sm.tsa.adfuller(ts_data)
    output = pd.Series(
        df_test[0:4], index=["Test statistic", "p-value", "used_lag", "NOBS"]
    )
    print(output)


check_stationary(df1)
check_stationary(df2)
check_stationary(df3)

Test statistic     -2.829267
p-value             0.054213
used_lag           12.000000
NOBS              130.000000
dtype: float64
Test statistic   -1.638423e+01
p-value           2.732892e-29
used_lag          1.100000e+01
NOBS              1.300000e+02
dtype: float64
Test statistic   -9.434675e+00
p-value           5.079967e-16
used_lag          1.400000e+01
NOBS              1.260000e+02
dtype: float64


fig = plt.figure(figsize=(10, 6))
ax1 = fig.add_subplot(211)
fig = sm.graphics.tsa.plot_acf(dataset["Passengers"].values.squeeze(), lags=20, ax=ax1)
ax2 = fig.add_subplot(212)
fig = sm.graphics.tsa.plot_pacf(dataset["Passengers"], lags=20, ax=ax2)
plt.show()


from statsmodels.tsa.seasonal import seasonal_decompose

data = dataset.iloc[:, 1:2]
result = seasonal_decompose(
    x=data, model="multiplicative", extrapolate_trend="freq", period=12
)

fig = result.plot()
plt.show()


dataset["Month"] = pd.to_datetime(dataset["Month"])
dataset = dataset.set_index(["Month"])
dataset.index = pd.DatetimeIndex(dataset.index.values, freq=dataset.index.inferred_freq)


train_data, test_data = (
    dataset[0 : int(len(data) * 0.9)],
    dataset[int(len(data) * 0.9) :],
)


from statsmodels.tsa.arima.model import ARIMA

model = ARIMA(
    train_data, order=(12, 2, 13), freq=dataset.index.inferred_freq
)  # (p,d,q)


from warnings import simplefilter

simplefilter("ignore")

model_fit = model.fit()


start_data = train_data.index[-1] + pd.DateOffset(months=1)
end_data = test_data.index[-1] + pd.DateOffset(months=len(test_data))


predictions = model_fit.predict(start=start_data, end=end_data)


import matplotlib.pyplot as plt

plt.figure(figsize=(10, 4))
plt.plot(train_data["Passengers"], color="green", label="Train")
plt.plot(test_data["Passengers"], color="red", label="Real")
plt.plot(predictions, color="blue", label="Predicted")
plt.title("ARIMA with optimal parameters")

plt.legend()
plt.show()


import math
from sklearn.metrics import mean_squared_error, mean_absolute_error


# report performance
mse = mean_squared_error(test_data["Passengers"], predictions[:15])
print("MSE: " + str(mse))
mae = mean_absolute_error(test_data["Passengers"], predictions[:15])
print("MAE: " + str(mae))
rmse = math.sqrt(mean_squared_error(test_data["Passengers"], predictions[:15]))
print("RMSE: " + str(rmse))

MSE: 264.3382033445668
MAE: 13.372375060851368
RMSE: 16.258480966700635


!pip install -q pmdarima

     ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 2.1/2.1 MB 32.4 MB/s eta 0:00:00


from pmdarima.arima import auto_arima

stepwise_model = auto_arima(
    train_data,
    start_p=1,
    start_q=1,
    max_p=3,
    max_q=3,
    m=12,
    start_P=0,
    seasonal=True,
    d=1,
    D=1,
    trace=True,
    error_action="ignore",
    suppress_warnings=True,
    stepwise=True,
)
print(stepwise_model.aic())

Performing stepwise search to minimize aic
 ARIMA(1,1,1)(0,1,1)[12]             : AIC=879.138, Time=0.46 sec
 ARIMA(0,1,0)(0,1,0)[12]             : AIC=881.901, Time=0.06 sec
 ARIMA(1,1,0)(1,1,0)[12]             : AIC=877.920, Time=0.27 sec
 ARIMA(0,1,1)(0,1,1)[12]             : AIC=878.938, Time=0.30 sec
 ARIMA(1,1,0)(0,1,0)[12]             : AIC=876.775, Time=0.10 sec
 ARIMA(1,1,0)(0,1,1)[12]             : AIC=878.101, Time=0.29 sec
 ARIMA(1,1,0)(1,1,1)[12]             : AIC=inf, Time=1.13 sec
 ARIMA(2,1,0)(0,1,0)[12]             : AIC=877.993, Time=0.07 sec
 ARIMA(1,1,1)(0,1,0)[12]             : AIC=877.677, Time=0.08 sec
 ARIMA(0,1,1)(0,1,0)[12]             : AIC=877.710, Time=0.06 sec
 ARIMA(2,1,1)(0,1,0)[12]             : AIC=879.650, Time=0.20 sec
 ARIMA(1,1,0)(0,1,0)[12] intercept   : AIC=878.414, Time=0.10 sec

Best model:  ARIMA(1,1,0)(0,1,0)[12]          
Total fit time: 3.135 seconds
876.7750530228392


stepwise_model.fit(train_data)

 ARIMA(1,1,0)(0,1,0)[12]

 ARIMA(1,1,0)(0,1,0)[12]


future_forecast = stepwise_model.predict(start=start_data, n_periods=(len(predictions)))


import matplotlib.pyplot as plt

plt.figure(figsize=(10, 4))
plt.plot(train_data["Passengers"], color="green", label="Train")
plt.plot(test_data["Passengers"], color="red", label="Real")
plt.plot(future_forecast, color="blue", label="Predicted")

plt.title("ARIMA with optimal parameters")
plt.xlabel("Time")
plt.ylabel("Passengers")
plt.legend()
plt.grid(True)
plt.show()


mse = mean_squared_error(test_data["Passengers"], future_forecast[: len(test_data)])
print("MSE: " + str(mse))
mae = mean_absolute_error(test_data["Passengers"], future_forecast[: len(test_data)])
print("MAE: " + str(mae))
rmse = math.sqrt(
    mean_squared_error(test_data["Passengers"], future_forecast[: len(test_data)])
)
print("RMSE: " + str(rmse))

MSE: 328.92348797154153
MAE: 13.990150688148209
RMSE: 18.136247902241013


import torch

torch.manual_seed(42)

rnn_cell = torch.nn.RNNCell(input_size=3, hidden_size=2)
dummy_sequence = torch.randn((1, 3))  # batch, input_size
h = rnn_cell(dummy_sequence)
print("Inital shape:".ljust(17), f"{dummy_sequence.shape}")
print("Resulting shape:".ljust(17), f"{h.shape}")  # hidden state

Inital shape:     torch.Size([1, 3])
Resulting shape:  torch.Size([1, 2])


from torch import nn


# Simple RNNcell without a bias and batch support
class SimplifiedRNNCell(nn.Module):
    def __init__(self, input_size, hidden_size):
        super().__init__()
        # Init weight matrix, for simplicity omit bias
        self.W_hx = (
            torch.randn(input_size, hidden_size) * 0.0001
        )  # hidden_size == number of neurons
        self.W_hh = (
            torch.randn(hidden_size, hidden_size) * 0.0001
        )  # naive initialization
        self.h0 = torch.zeros((hidden_size))  # Initial hidden state

    def forward(self, x, h=None):  # Without a batch dimension
        if h is None:
            h = self.h0
        h = torch.tanh(torch.matmul(self.W_hx.T, x) + torch.matmul(self.W_hh.T, h))
        return h


simple_rnn_cell = SimplifiedRNNCell(input_size=3, hidden_size=2)
h = simple_rnn_cell(dummy_sequence[0])  # No batch
print(f"Out = h\n{h.shape} \n{h}")

Out = h
torch.Size([2]) 
tensor([-3.6047e-05, -7.6246e-05])


rnn = torch.nn.RNN(input_size=3, hidden_size=2, batch_first=False)  # batch_first = True
dummy_batched_seq = torch.randn((2, 1, 3))  # seq_len, batch, input_size
out, h = rnn(dummy_batched_seq)

print("Inital shape:".ljust(20), f"{dummy_batched_seq.shape}")
print("Resulting shape:".ljust(20), f"{out.shape}")
print("Hidden state shape:".ljust(20), f"{h.shape}")

Inital shape:        torch.Size([2, 1, 3])
Resulting shape:     torch.Size([2, 1, 2])
Hidden state shape:  torch.Size([1, 1, 2])


import numpy as np


# Simple RNN without batching
class SimplifiedRNNLayer(nn.Module):
    def __init__(self, input_size, hidden_size):
        super().__init__()
        self.rnn_cell = SimplifiedRNNCell(input_size, hidden_size)

    # Without a batch dimension x have shape seq_len * input_size
    def forward(self, x, h=None):
        all_h = []
        for i in range(x.shape[0]):  # iterating over timestamps
            h = self.rnn_cell(torch.Tensor(x[i]), h)
            all_h.append(h)
        return np.stack(all_h), h


simple_rnn = SimplifiedRNNLayer(input_size=4, hidden_size=2)

sequence = np.array(
    [[0, 1, 2, 0], [3, 4, 5, 0]]
)  # batch with one sequence of two elements

out, h = simple_rnn(sequence)
print("Inital shape:".ljust(20), f"{sequence.shape}")
print("Resulting shape:".ljust(20), f"{out.shape}")
print("Hidden state shape:".ljust(20), f"{h.shape}")

Inital shape:        (2, 4)
Resulting shape:     (2, 2)
Hidden state shape:  torch.Size([2])


dummy_seq = torch.randn((2, 1, 3))  #  seq_len, batch, input_size

print("RNNCell")
rnn_cell = torch.nn.RNNCell(3, 2)
print("Parameter".ljust(10), "Shape")
for t, p in rnn_cell.named_parameters():
    print(t.ljust(10), p.shape)

cell_out = rnn_cell(dummy_seq[0, :, :])  # take first element from sequence
print()
print("Result shape =".ljust(20), cell_out.shape)
print("Hidden state shape =".ljust(20), cell_out.shape)  # one hidden state

print("----------------------------------------")

print("RNN")
rnn = torch.nn.RNN(3, 2)
print("Parameter".ljust(15), "Shape")
for t, p in rnn.named_parameters():
    print(t.ljust(15), p.shape)

out, h = rnn(dummy_seq)

print()
print("Result shape =".ljust(20), out.shape)  # h for all timestamps element
print("Hidden state shape =".ljust(20), cell_out.shape)  # h for last element

RNNCell
Parameter  Shape
weight_ih  torch.Size([2, 3])
weight_hh  torch.Size([2, 2])
bias_ih    torch.Size([2])
bias_hh    torch.Size([2])

Result shape =       torch.Size([1, 2])
Hidden state shape = torch.Size([1, 2])
----------------------------------------
RNN
Parameter       Shape
weight_ih_l0    torch.Size([2, 3])
weight_hh_l0    torch.Size([2, 2])
bias_ih_l0      torch.Size([2])
bias_hh_l0      torch.Size([2])

Result shape =       torch.Size([2, 1, 2])
Hidden state shape = torch.Size([1, 2])


dummy_input = torch.randn((2, 1, 3))  # seq_len, batch, input_size
rnn = torch.nn.RNN(3, 2, num_layers=3)

# Weights matrix sizes not changed!
for t, p in rnn.named_parameters():
    print(t, p.shape)

out, h = rnn(dummy_input)

print()
print("Out:\n", out.shape)  # Hidden states for all elements from top layer
print("h:\n", h.shape)  # Hidden states for last element for all layers

weight_ih_l0 torch.Size([2, 3])
weight_hh_l0 torch.Size([2, 2])
bias_ih_l0 torch.Size([2])
bias_hh_l0 torch.Size([2])
weight_ih_l1 torch.Size([2, 2])
weight_hh_l1 torch.Size([2, 2])
bias_ih_l1 torch.Size([2])
bias_hh_l1 torch.Size([2])
weight_ih_l2 torch.Size([2, 2])
weight_hh_l2 torch.Size([2, 2])
bias_ih_l2 torch.Size([2])
bias_hh_l2 torch.Size([2])

Out:
 torch.Size([2, 1, 2])
h:
 torch.Size([3, 1, 2])


import pandas as pd

dataset = pd.read_csv(
    "https://edunet.kea.su/repo/EduNet-web_dependencies/datasets/airline-passengers.csv"
)

training_data = dataset.iloc[:, 1:2].values  # transform dataframe to numpy.array


# Min-Max normalization
td_min = training_data.min()
td_max = training_data.max()
print("Initial statistics:")
print("Minimum value:", repr(td_min).rjust(5))
print("Maximum value:", repr(td_max).rjust(5))

training_data = (training_data - td_min) / (td_max - td_min)
print("\nResulting statistics:")
print("Minimum value:", repr(training_data.min()).rjust(5))
print("Maximum value:", repr(training_data.max()).rjust(5))

Initial statistics:
Minimum value:   104
Maximum value:   622

Resulting statistics:
Minimum value:   0.0
Maximum value:   1.0


import torch
import numpy as np


def sliding_windows(data, seq_length):
    x = []
    y = []

    for i in range(len(data) - seq_length):
        _x = data[i : (i + seq_length)]  # picking several sequential observations
        _y = data[i + seq_length]  # picking the subsequent observation
        x.append(_x)
        y.append(_y)

    return torch.Tensor(np.array(x)), torch.Tensor(np.array(y))


# set length of the ensemble; accuracy of the predictions and
# speed perfomance almost always depend on it size
seq_length = 8  # compare 2 and 32
x, y = sliding_windows(training_data, seq_length)
print("Example of the obtained data:\n")
print("Data corresponding to the first x:")
print(x[0])
print("Data corresponding to the first y:")
print(y[0])

Example of the obtained data:

Data corresponding to the first x:
tensor([[0.0154],
        [0.0270],
        [0.0541],
        [0.0483],
        [0.0328],
        [0.0598],
        [0.0849],
        [0.0849]])
Data corresponding to the first y:
tensor([0.0618])


train_size = int(len(y) * 0.8)

x_train = x[:train_size]
y_train = y[:train_size]

x_test = x[train_size + seq_length :]
y_test = y[train_size + seq_length :]

print("Train data:")
print("x shape:", x_train.shape)
print("y shape:", y_train.shape)

print("\nTest data:")
print("x shape:", x_test.shape)
print("y shape:", y_test.shape)

Train data:
x shape: torch.Size([108, 8, 1])
y shape: torch.Size([108, 1])

Test data:
x shape: torch.Size([20, 8, 1])
y shape: torch.Size([20, 1])


import torch.nn as nn


class AirTrafficPredictor(nn.Module):
    def __init__(self, input_size, hidden_size):
        # hidden_size == number of neurons
        super().__init__()
        self.rnn = nn.RNN(
            input_size=input_size, hidden_size=hidden_size, batch_first=True
        )
        self.fc = nn.Linear(hidden_size, 1)  # Predict only one value

    def forward(self, x):
        # print("x: ",x.shape) # 108 x 8 x 1 : [batch_size, seq_len, input_size]
        out, h = self.rnn(x)
        # print("out: ", out.shape) # 108 x 8 x 4 : [batch_size, seq_len, hidden_size] Useless!
        # print("h : ", h.shape) # 1 x 108 x 4 [ num_layers, batch_size, hidden_size]
        y = self.fc(h)
        # print("y",y.shape) # 1 x 108 x 1
        return y, h


def time_series_train(model, num_epochs=2000, learning_rate=0.01):
    criterion = torch.nn.MSELoss()  # mean-squared error for regression
    optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)

    # Train the model
    for epoch in range(num_epochs):
        y_pred, h = model(x_train)  # we don't use h there, but we can!
        optimizer.zero_grad()

        # obtain the loss
        loss = criterion(y_pred[0], y_train)  # for shape compatibility
        loss.backward()

        optimizer.step()
        if epoch % 100 == 0:
            print(f"Epoch: {epoch},".ljust(15), "loss: %1.5f" % (loss.item()))


print("Simple RNN training process with MSE loss:")
input_size = 1
hidden_size = 8
rnn = AirTrafficPredictor(input_size, hidden_size)
time_series_train(rnn)

Simple RNN training process with MSE loss:
Epoch: 0,       loss: 0.01983
Epoch: 100,     loss: 0.00258
Epoch: 200,     loss: 0.00223
Epoch: 300,     loss: 0.00208
Epoch: 400,     loss: 0.00205
Epoch: 500,     loss: 0.00204
Epoch: 600,     loss: 0.00202
Epoch: 700,     loss: 0.00211
Epoch: 800,     loss: 0.00196
Epoch: 900,     loss: 0.00189
Epoch: 1000,    loss: 0.00181
Epoch: 1100,    loss: 0.00170
Epoch: 1200,    loss: 0.00150
Epoch: 1300,    loss: 0.00128
Epoch: 1400,    loss: 0.00103
Epoch: 1500,    loss: 0.00095
Epoch: 1600,    loss: 0.00092
Epoch: 1700,    loss: 0.00091
Epoch: 1800,    loss: 0.00089
Epoch: 1900,    loss: 0.00087


import matplotlib.pyplot as plt


def time_series_plot(train_predict):
    data_predict = train_predict.data
    y_data_plot = y.data

    # Denormalize
    data_predict = data_predict[0] * (td_max - td_min) + td_min
    y_data_plot = y_data_plot * (td_max - td_min) + td_min

    # Plotting
    plt.figure(figsize=(12, 4))
    plt.axvline(x=train_size + seq_length, c="r", linestyle="--")
    # shifting the curve as first y-value not correspond first value overall
    plt.plot(np.arange(y_data_plot.shape[0]), y_data_plot)

    plt.plot(np.arange(y_data_plot.shape[0]), data_predict)

    plt.title("Number of passengers per month")
    plt.ylabel("#passengers")
    plt.xlabel("Month")
    # plt.xticks(labels_to_display, dataset["Month"][labels_to_display])

    plt.legend(["Train/Test separation", "Real", "Predicted"])
    plt.grid(axis="x")
    plt.show()


rnn.eval()
train_predict, h = rnn(x)
time_series_plot(train_predict)


import torch

lstm_cell = torch.nn.LSTMCell(input_size=3, hidden_size=4)
input = torch.randn(1, 3)  # batch, input_size
h_0 = torch.randn(1, 4)
c_0 = torch.randn(1, 4)
h, c = lstm_cell(input, (h_0, c_0))  # second arg is tuple
print("Shape of h:", h.shape)  # batch, hidden_size
print("Shape of c:", c.shape)  # batch, hidden_size

Shape of h: torch.Size([1, 4])
Shape of c: torch.Size([1, 4])


import torch.nn as nn

lstm = nn.LSTM(input_size=4, hidden_size=5)
input = torch.randn(3, 2, 4)  # seq_len, batch, input_size
out, (h, c) = lstm(input)  # h and c returned in tuple

print("Input shape:".ljust(15), input.shape)
print("Shape of h".ljust(15), h.shape)  # batch, hidden_size
print("Shape of c".ljust(15), c.shape)  # batch, hidden_size
print(
    "Output shape:".ljust(15), out.shape
)  # seq_len, batch, hidden_size : h for each element

Input shape:    torch.Size([3, 2, 4])
Shape of h      torch.Size([1, 2, 5])
Shape of c      torch.Size([1, 2, 5])
Output shape:   torch.Size([3, 2, 5])


import torch
import random
import numpy as np


def set_random_seed(seed):
    torch.manual_seed(seed)
    torch.cuda.manual_seed(seed)
    np.random.seed(seed)
    random.seed(seed)


set_random_seed(42)


import numpy as np

N = 30  # number of samples
L = 300  # length of each sample (number of values for each sine wave)
T = 10  # width of the wave
x = np.empty((N, L), np.float32)  # instantiate empty array
x[:] = np.arange(L) + np.random.randint(-4 * T, 4 * T, N).reshape(N, 1)
y = np.sin(x / 1.0 / T).astype(np.float32)


import matplotlib.pyplot as plt

plt.plot(y[0], label="Sequense1")
plt.plot(y[1], label="Sequense2")
plt.xlabel("x")
plt.ylabel("y")
plt.legend(loc="upper left")

plt.show()


import torch
import torch.nn as nn


class LSTM(nn.Module):
    def __init__(self, hidden_state=512):
        super(LSTM, self).__init__()
        self.hidden_state = hidden_state
        # lstm1, lstm2, linear are all layers in the network
        self.lstm1 = nn.LSTMCell(1, self.hidden_state)
        self.lstm2 = nn.LSTMCell(self.hidden_state, self.hidden_state)
        self.linear = nn.Linear(self.hidden_state, 1)

    def forward(self, y, future_preds=0):
        outputs, n_samples = [], y.size(0)
        h_t = torch.zeros(n_samples, self.hidden_state, dtype=torch.float32)
        c_t = torch.zeros(n_samples, self.hidden_state, dtype=torch.float32)
        h_t2 = torch.zeros(n_samples, self.hidden_state, dtype=torch.float32)
        c_t2 = torch.zeros(n_samples, self.hidden_state, dtype=torch.float32)

        for time_step in y.split(1, dim=1):
            # N, 1
            h_t, c_t = self.lstm1(
                time_step, (h_t, c_t)
            )  # initial hidden and cell states
            h_t2, c_t2 = self.lstm2(h_t, (h_t2, c_t2))  # new hidden and cell states
            output = self.linear(h_t2)  # output from the last FC layer
            outputs.append(output)

        for i in range(future_preds):
            # this only generates future predictions if we pass in future_preds>0
            # mirrors the code above, using last output/prediction as input
            h_t, c_t = self.lstm1(output, (h_t, c_t))
            h_t2, c_t2 = self.lstm2(h_t, (h_t2, c_t2))
            output = self.linear(h_t2)
            outputs.append(output)
        # transform list to tensor
        outputs = torch.cat(outputs, dim=1)
        return outputs


a = torch.from_numpy(y[3:, :-1])
b = a.split(1, dim=1)
b[0].shape

torch.Size([27, 1])


train_input = torch.from_numpy(y[3:, :-1])  # (27, 299)
train_target = torch.from_numpy(y[3:, 1:])  # (27, 299)
test_input = torch.from_numpy(y[:3, :-1])  # (3, 299)
test_target = torch.from_numpy(y[:3, 1:])  # (3, 299)


def draw(yi, n, i, future):
    f, (ax1, ax2, ax3) = plt.subplots(1, 3, sharey=True, figsize=(12, 3))

    plt.title(f"Step {i+1}")
    plt.xlabel("x")
    plt.ylabel("y")

    ax1.plot(np.arange(n), yi[0][:n], "r", linewidth=2.0)
    ax1.plot(np.arange(n, n + future), yi[0][n:], "r" + ":", linewidth=2.0)

    ax2.plot(np.arange(n), yi[1][:n], "g", linewidth=2.0)
    ax2.plot(np.arange(n, n + future), yi[1][n:], "g" + ":", linewidth=2.0)

    ax3.plot(np.arange(n), yi[2][:n], "b", linewidth=2.0)
    ax3.plot(np.arange(n, n + future), yi[2][n:], "b" + ":", linewidth=2.0)

    plt.savefig("predict%d.png" % i, dpi=200)
    plt.show()
    plt.close()


def training_loop(
    num_epochs,
    model,
    optimiser,
    loss_fn,
    train_input,
    train_target,
    test_input,
    test_target,
):
    for i in range(num_epochs):

        def closure():
            optimiser.zero_grad()
            out = model(train_input)
            loss = loss_fn(out, train_target)
            loss.backward()
            return loss

        optimiser.step(closure)
        with torch.no_grad():
            future = 100
            pred = model(test_input, future_preds=future)
            # use all pred samples, but only go to 299
            loss = loss_fn(pred[:, :-future], test_target)
            y = pred.detach().numpy()
        # draw figures
        n = train_input.shape[1]  # 299
        draw(y, n, i, future)

        # print the loss
        out = model(train_input)
        loss_print = loss_fn(out, train_target)
        print("Step: {}, Loss: {}".format(i, loss_print))


import torch.optim as optim


model = LSTM(hidden_state=128)
criterion = nn.MSELoss()
optimiser = optim.LBFGS(model.parameters(), lr=0.01)

num_epochs = 20

training_loop(
    num_epochs,
    model,
    optimiser,
    criterion,
    train_input,
    train_target,
    test_input,
    test_target,
)

Step: 0, Loss: 0.16092360019683838

Step: 1, Loss: 0.0824916735291481

Step: 2, Loss: 0.05598531663417816

Step: 3, Loss: 0.037580814212560654

Step: 4, Loss: 0.020968766883015633

Step: 5, Loss: 0.008734678849577904

Step: 6, Loss: 0.005881512071937323

Step: 7, Loss: 0.004695284180343151

Step: 8, Loss: 0.004001234192401171

Step: 9, Loss: 0.003534492803737521


dummy_input = torch.randn((2, 1, 3))  # seq_len, batch, input_size
rnn = torch.nn.RNN(3, 2, bidirectional=True)

for t, p in rnn.named_parameters():
    print(t, p.shape)

out, h = rnn(dummy_input)

# Concatenated Hidden states from both layers
print("Out:\n", out.shape)
# Hidden states last element from  both : 2*num_layers*hidden_state
print("h:\n", h.shape)

weight_ih_l0 torch.Size([2, 3])
weight_hh_l0 torch.Size([2, 2])
bias_ih_l0 torch.Size([2])
bias_hh_l0 torch.Size([2])
weight_ih_l0_reverse torch.Size([2, 3])
weight_hh_l0_reverse torch.Size([2, 2])
bias_ih_l0_reverse torch.Size([2])
bias_hh_l0_reverse torch.Size([2])
Out:
 torch.Size([2, 1, 4])
h:
 torch.Size([2, 1, 2])


gru = torch.nn.GRU(input_size=4, hidden_size=3)
input = torch.randn(2, 1, 4)  # seq_len, batch, input_size
h0 = torch.randn(1, 1, 3)
output, h = gru(input, h0)

print("Input shape:".ljust(15), input.shape)
print("Shape of h:".ljust(15), h.shape)  # last h
print("Output shape:".ljust(15), output.shape)  # seq_len = 2

Input shape:    torch.Size([2, 1, 4])
Shape of h:     torch.Size([1, 1, 3])
Output shape:   torch.Size([2, 1, 3])


import pprint

text = ["hey how are you", "good i am fine", "have a nice day"]

# Join all the sentences together and extract the unique characters
# from the combined sentences
chars = set("".join(text))
# Creating a dictionary that maps integers to the characters
int2char = dict(enumerate(chars))
# Creating another dictionary that maps characters to integers
char2int = {char: ind for ind, char in int2char.items()}

print("Dictionary for mapping character to the integer:")
pprint.pprint(char2int)

Dictionary for mapping character to the integer:
{' ': 2,
 'a': 15,
 'c': 6,
 'd': 16,
 'e': 13,
 'f': 12,
 'g': 10,
 'h': 4,
 'i': 14,
 'm': 9,
 'n': 3,
 'o': 0,
 'r': 8,
 'u': 11,
 'v': 5,
 'w': 1,
 'y': 7}


lengths = [len(sent) for sent in text]
maxlen = max(lengths)
print(f"The longest string has {maxlen} characters.\n")

print(f"Initial texts:\n{text}")
# A simple loop that loops through the list of sentences and adds
# a ' ' whitespace until the length of the sentence matches
# the length of the longest sentence
for i in range(len(text)):
    while len(text[i]) < maxlen:
        text[i] += " "

print(f"Resulting texts:\n{text}")

The longest string has 15 characters.

Initial texts:
['hey how are you', 'good i am fine', 'have a nice day']
Resulting texts:
['hey how are you', 'good i am fine ', 'have a nice day']


# Creating lists that will hold our input and target sequences
input_seq = []
target_seq = []

for i in range(len(text)):
    # Remove last character for input sequence
    input_seq.append(text[i][:-1])

    # Remove first character for target sequence
    target_seq.append(text[i][1:])

    print("Input sequence:".ljust(18), f"'{input_seq[i]}'")
    print("Target sequence:".ljust(18), f"'{target_seq[i]}'")
    print()

Input sequence:    'hey how are yo'
Target sequence:   'ey how are you'

Input sequence:    'good i am fine'
Target sequence:   'ood i am fine '

Input sequence:    'have a nice da'
Target sequence:   'ave a nice day'


for i in range(len(text)):
    input_seq[i] = [char2int[character] for character in input_seq[i]]
    target_seq[i] = [char2int[character] for character in target_seq[i]]

    print("Encoded input sequence:".ljust(25), input_seq[i])
    print("Encoded target sequence:".ljust(25), target_seq[i])
    print()

Encoded input sequence:   [4, 13, 7, 2, 4, 0, 1, 2, 15, 8, 13, 2, 7, 0]
Encoded target sequence:  [13, 7, 2, 4, 0, 1, 2, 15, 8, 13, 2, 7, 0, 11]

Encoded input sequence:   [10, 0, 0, 16, 2, 14, 2, 15, 9, 2, 12, 14, 3, 13]
Encoded target sequence:  [0, 0, 16, 2, 14, 2, 15, 9, 2, 12, 14, 3, 13, 2]

Encoded input sequence:   [4, 15, 5, 13, 2, 15, 2, 3, 14, 6, 13, 2, 16, 15]
Encoded target sequence:  [15, 5, 13, 2, 15, 2, 3, 14, 6, 13, 2, 16, 15, 7]


import numpy as np

dict_size = len(char2int)
seq_len = maxlen - 1
batch_size = len(text)


def one_hot_encode(sequence, dict_size, seq_len, batch_size):
    # Creating a multi-dimensional array of zeros with the desired output shape
    features = np.zeros((batch_size, seq_len, dict_size), dtype=np.float32)

    # Replacing the 0 at the relevant character index with a 1 to represent that character
    for i in range(batch_size):
        for u in range(seq_len):
            features[i, u, sequence[i][u]] = 1
    return features


input_seq = one_hot_encode(input_seq, dict_size, seq_len, batch_size)
print(
    "Input shape: {} --> (Batch Size, Sequence Length, One-Hot Encoding Size)".format(
        input_seq.shape
    )
)
print(input_seq[0])

Input shape: (3, 14, 17) --> (Batch Size, Sequence Length, One-Hot Encoding Size)
[[0. 0. 0. 0. 1. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.]
 [0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 1. 0. 0. 0.]
 [0. 0. 0. 0. 0. 0. 0. 1. 0. 0. 0. 0. 0. 0. 0. 0. 0.]
 [0. 0. 1. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.]
 [0. 0. 0. 0. 1. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.]
 [1. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.]
 [0. 1. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.]
 [0. 0. 1. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.]
 [0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 1. 0.]
 [0. 0. 0. 0. 0. 0. 0. 0. 1. 0. 0. 0. 0. 0. 0. 0. 0.]
 [0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 1. 0. 0. 0.]
 [0. 0. 1. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.]
 [0. 0. 0. 0. 0. 0. 0. 1. 0. 0. 0. 0. 0. 0. 0. 0. 0.]
 [1. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.]]


# Convert data to tensor
import torch

input_seq = torch.Tensor(input_seq)
target_seq = torch.Tensor(target_seq)


import torch.nn as nn


class NextCharacterGenerator(nn.Module):
    def __init__(self, input_size, output_size, hidden_dim, n_layers):
        super().__init__()

        # RNN Layer
        self.rnn = nn.RNN(input_size, hidden_size=hidden_dim, batch_first=True)
        # Fully connected layer
        self.fc = nn.Linear(hidden_dim, output_size)

    def forward(self, x):
        batch_size = x.size(0)
        # Initializing hidden state for first input using method defined below
        hidden_0 = torch.zeros(
            1, batch_size, self.rnn.hidden_size
        )  # 1 correspond to number of layers

        # Passing in the input and hidden state into the model and obtaining outputs
        out, hidden = self.rnn(x, hidden_0)

        # Reshaping the outputs such that it can be fit into the fully connected layer
        # Need Only if n_layers > 1
        out = out.contiguous().view(-1, self.rnn.hidden_size)
        out = self.fc(out)

        return out, hidden


# Instantiate the model with hyperparameters
model = NextCharacterGenerator(
    input_size=dict_size, output_size=dict_size, hidden_dim=12, n_layers=1
)

# Define hyperparameters
num_epochs = 100

# Define Loss, Optimizer
criterion = nn.CrossEntropyLoss()
optimizer = torch.optim.Adam(model.parameters(), lr=0.01)

# Training Run
for epoch in range(1, num_epochs + 1):
    optimizer.zero_grad()  # Clears existing gradients from previous epoch
    output, hidden = model(input_seq)
    loss = criterion(output, target_seq.view(-1).long())
    loss.backward()  # Does backpropagation and calculates gradients
    optimizer.step()  # Updates the weights accordingly

    if epoch % 10 == 0:
        print(f"Epoch: {epoch}/{num_epochs}".ljust(20), end=" ")
        print("Loss: {:.4f}".format(loss.item()))

Epoch: 10/100        Loss: 2.4534
Epoch: 20/100        Loss: 2.1249
Epoch: 30/100        Loss: 1.7164
Epoch: 40/100        Loss: 1.3150
Epoch: 50/100        Loss: 0.9692
Epoch: 60/100        Loss: 0.6836
Epoch: 70/100        Loss: 0.4718
Epoch: 80/100        Loss: 0.3277
Epoch: 90/100        Loss: 0.2334
Epoch: 100/100       Loss: 0.1742


def predict(model, character):
    # One-hot encoding our input to fit into the model
    character = np.array([[char2int[c] for c in character]])
    character = one_hot_encode(character, dict_size, character.shape[1], 1)
    character = torch.from_numpy(character)

    out, hidden = model(character)
    # print(out.shape)
    # print(out)
    prob = nn.functional.softmax(out[-1], dim=0).data
    # Taking the class with the highest probability score from the output
    char_ind = torch.max(prob, dim=0)[1].item()

    return int2char[char_ind], hidden


def sample(model, out_len, start="hey"):
    model.eval()  # eval mode
    start = start.lower()
    # First off, run through the starting characters
    chars = [ch for ch in start]
    size = out_len - len(chars)
    # Now pass in the previous characters and get a new one
    for _ in range(size):
        char, h = predict(model, chars)
        chars.append(char)

    return "".join(chars)


sample(model, 15, "good")

'good i am fine '


for _ in range(3):
    print(sample(model, 15, "good"))

good i am fine 
good i am fine 
good i am fine


from sklearn.feature_extraction.text import TfidfVectorizer

corpus = [
    "This is the first document.",
    "This document is the second document.",
    "And this is the third one.",
    "Is this the first document?",
]

vectorizer = TfidfVectorizer()
x = vectorizer.fit_transform(corpus)

print("Tf-idf dictionary:", vectorizer.get_feature_names_out())
print("Tf-idf dictionary len:", len(vectorizer.get_feature_names_out()))
print("Tf-idf shape:", x.shape)
print("Tf-idf type:", type(x))
print("Tf-idf values:", x)

Tf-idf dictionary: ['and' 'document' 'first' 'is' 'one' 'second' 'the' 'third' 'this']
Tf-idf dictionary len: 9
Tf-idf shape: (4, 9)
Tf-idf type: <class 'scipy.sparse._csr.csr_matrix'>
Tf-idf values:   (0, 1)	0.46979138557992045
  (0, 2)	0.5802858236844359
  (0, 6)	0.38408524091481483
  (0, 3)	0.38408524091481483
  (0, 8)	0.38408524091481483
  (1, 5)	0.5386476208856763
  (1, 1)	0.6876235979836938
  (1, 6)	0.281088674033753
  (1, 3)	0.281088674033753
  (1, 8)	0.281088674033753
  (2, 4)	0.511848512707169
  (2, 7)	0.511848512707169
  (2, 0)	0.511848512707169
  (2, 6)	0.267103787642168
  (2, 3)	0.267103787642168
  (2, 8)	0.267103787642168
  (3, 1)	0.46979138557992045
  (3, 2)	0.5802858236844359
  (3, 6)	0.38408524091481483
  (3, 3)	0.38408524091481483
  (3, 8)	0.38408524091481483


# Let's say you have 2 sentences (lowercased, punctuations removed):
sentences = "i am new to pytorch i am having fun"

words = sentences.split(" ")

print(f"All words: {words} \n")

vocab = set(words)  # create a vocabulary
vocab_size = len(vocab)

print(f"Vocabulary (unique words): {vocab} \n")
print(f"Vocabulary size: {vocab_size} \n")

# map words to unique indices
word2idx = {word: ind for ind, word in enumerate(vocab)}

print(f"Word-to-id dictionary: {word2idx} \n")

encoded_sentences = [word2idx[word] for word in words]

print(f"Encoded sentences: {encoded_sentences}")

# let's say you want embedding dimension to be 3
emb_dim = 3

All words: ['i', 'am', 'new', 'to', 'pytorch', 'i', 'am', 'having', 'fun'] 

Vocabulary (unique words): {'new', 'am', 'pytorch', 'having', 'fun', 'i', 'to'} 

Vocabulary size: 7 

Word-to-id dictionary: {'new': 0, 'am': 1, 'pytorch': 2, 'having': 3, 'fun': 4, 'i': 5, 'to': 6} 

Encoded sentences: [5, 1, 0, 6, 2, 5, 1, 3, 4]


import torch
import torch.nn as nn


emb_layer = nn.Embedding(vocab_size, emb_dim)
word_vectors = emb_layer(torch.LongTensor(encoded_sentences))

print(f"Shape of encoded sentences: {word_vectors.shape} \n")
print(f"Shape of weigths: {emb_layer.weight.shape}")

Shape of encoded sentences: torch.Size([9, 3]) 

Shape of weigths: torch.Size([7, 3])


emb_layer.weight.requires_grad

True


emb_layer.weight.requires_grad = False


# predefined weights
weight = torch.FloatTensor([[0.1, 0.2, 0.3], [0.4, 0.5, 0.6]])
print(weight.shape)
embedding = nn.Embedding.from_pretrained(weight)
# get embeddings for ind 0 and 1
embedding(torch.LongTensor([0, 1]))

torch.Size([2, 3])

tensor([[0.1000, 0.2000, 0.3000],
        [0.4000, 0.5000, 0.6000]])


# Source: https://drive.google.com/file/d/0B7XkCwpI5KDYNlNUTTlSS21pQmM/edit?resourcekey=0-wjGZdNAUop6WykTtMip30g
!wget -q https://edunet.kea.su/repo/EduNet-web_dependencies/weights/GoogleNews-vectors-negative300.bin.gz
!gunzip -q GoogleNews-vectors-negative300.bin.gz


from gensim.models import KeyedVectors

wordvector_path = "GoogleNews-vectors-negative300.bin"
word_vectors = KeyedVectors.load_word2vec_format(wordvector_path, binary=True)


weights = torch.FloatTensor(word_vectors.vectors)


weights.shape

torch.Size([3000000, 300])


embedding = nn.Embedding.from_pretrained(weight)

input = torch.LongTensor([0, 1])

embedding(input)

tensor([[0.1000, 0.2000, 0.3000],
        [0.4000, 0.5000, 0.6000]])


import torchtext

glove = torchtext.vocab.GloVe(
    name="6B", dim=50, max_vectors=10000
)  # use 10k most common words

.vector_cache/glove.6B.zip: 862MB [02:44, 5.25MB/s]                           
100%|█████████▉| 9999/10000 [00:00<00:00, 31607.97it/s]


glove_emb = nn.Embedding.from_pretrained(glove.vectors)


input = torch.LongTensor([0, 1])
glove_emb(input).shape

torch.Size([2, 50])


class RNN_with_Embedding_Layer(nn.Module):
    def __init__(self, input_size, hidden_size, num_classes):
        super(RNN_with_Embedding_Layer, self).__init__()
        self.emb = nn.Embedding.from_pretrained(glove.vectors)
        self.hidden_size = hidden_size
        self.rnn = nn.RNN(input_size, hidden_size, batch_first=True)
        self.fc = nn.Linear(hidden_size, num_classes)

    def forward(self, x):
        # Look up the embedding
        x = self.emb(x)
        # Set an initial hidden state
        h0 = torch.zeros(1, x.size(0), self.hidden_size)
        # Forward propagate the RNN
        out, _ = self.rnn(x, h0)
        # Pass the output of the last time step to the classifier
        out = self.fc(out[:, -1, :])
        return out


model = RNN_with_Embedding_Layer(input_size=50, hidden_size=128, num_classes=3)
print(model)

RNN_with_Embedding_Layer(
  (emb): Embedding(10000, 50)
  (rnn): RNN(50, 128, batch_first=True)
  (fc): Linear(in_features=128, out_features=3, bias=True)
)


!pip install -q audiomentations
!wget -q https://edunet.kea.su/repo/EduNet-web_dependencies/dev-2.0/L09/audio_example.wav

     ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 82.3/82.3 kB 2.7 MB/s eta 0:00:00


from IPython.display import Audio

# Get input audio
input_audio = "/content/audio_example.wav"

display(Audio(input_audio))


import librosa

data, sr = librosa.load("/content/audio_example.wav")  # sr - sampling rate


from audiomentations import AddGaussianSNR

augment = AddGaussianSNR(min_snr_in_db=3, max_snr_in_db=7, p=1)

# Augment/transform the audio data
augmented_data = augment(samples=data, sample_rate=sr)

display(Audio(augmented_data, rate=sr))


import numpy as np
from scipy.signal import spectrogram
from matplotlib import pyplot as plt


def produce_plots(input_audio_arr, aug_audio, sr):
    f, t, Sxx_in = spectrogram(
        input_audio_arr, fs=sr
    )  # Compute spectrogram for the original signal (f - frequency, t - time)
    f, t, Sxx_aug = spectrogram(aug_audio, fs=sr)

    fig, ax = plt.subplots(nrows=2, ncols=2, figsize=(20, 5))

    ax[0, 0].plot(input_audio_arr)
    ax[0, 0].set_xlim(0, len(input_audio_arr))
    ax[0, 0].set_xticks([])
    ax[0, 0].set_title("Original audio")

    ax[0, 1].plot(aug_audio)
    ax[0, 1].set_xlim(0, len(input_audio_arr))
    ax[0, 1].set_xticks([])
    ax[0, 1].set_title("Augmented  audio")

    ax[1, 0].imshow(
        np.log(Sxx_in),
        extent=[t.min(), t.max(), f.min(), f.max()],
        aspect="auto",
        cmap="inferno",
    )
    ax[1, 0].set_ylabel("Frequecny, Hz")
    ax[1, 0].set_xlabel("Time,s")

    ax[1, 1].imshow(
        np.log(Sxx_aug, where=Sxx_aug > 0),
        extent=[t.min(), t.max(), f.min(), f.max()],
        aspect="auto",
        cmap="inferno",
    )
    ax[1, 1].set_ylabel("Frequecny, Hz")
    ax[1, 1].set_xlabel("Time,s")

    plt.subplots_adjust(hspace=0)
    plt.show()


produce_plots(data, augmented_data, sr)


from audiomentations import TimeStretch

augment = TimeStretch(min_rate=0.8, max_rate=1.5, p=1)
augmented_data = augment(data, sample_rate=sr)

display(Audio(augmented_data, rate=sr))


produce_plots(data, augmented_data, sr)


from audiomentations import PitchShift

augment = PitchShift(min_semitones=1, max_semitones=12, p=1)
augmented_data = augment(data, sample_rate=sr)

display(Audio(augmented_data, rate=sr))


from audiomentations import Compose, AddGaussianNoise

augment = Compose(
    [
        AddGaussianNoise(min_amplitude=0.001, max_amplitude=0.015, p=1),
        TimeStretch(min_rate=0.8, max_rate=1.25, p=1),
        PitchShift(min_semitones=-4, max_semitones=4, p=1),
    ]
)

augmented_data = augment(data, sample_rate=sr)

display(Audio(augmented_data, rate=sr))


produce_plots(data, augmented_data, sr)


!pip install -q nlpaug

     ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 410.5/410.5 kB 9.1 MB/s eta 0:00:00


# Define input text
text = "Hello, future of AI for Science! How are you today?"
print(f"input text: {text}")

input text: Hello, future of AI for Science! How are you today?


import nlpaug.augmenter.char as nac

augment = nac.OcrAug()
augmented_text = augment.augment(text)

print(f"Original:\n{text}")
print(f"Augmented Texts:\n{augmented_text}")

Original:
Hello, future of AI for Science! How are you today?
Augmented Texts:
['Hello, future uf AI for Science! H0w are y0o tuday?']


augment = nac.KeyboardAug()
augmented_text = augment.augment(text)

print(f"Original:\n{text}")
print(f"Augmented Texts:\n{augmented_text}")

Original:
Hello, future of AI for Science! How are you today?
Augmented Texts:
['melPo, fu%uge of AI for DciSmce! How are you tkFay?']


import nlpaug.augmenter.word as naw

augment = naw.SpellingAug()
augmented_text = augment.augment(text, n=3)

print(f"Original:\n{text}")
print(f"Augmented Texts:\n{augmented_text}")

Original:
Hello, future of AI for Science! How are you today?
Augmented Texts:
['Hllo, future or AI dor Science! How arre you today?', 'Hello, future of AI fom Sciece! Gow ares you today?', 'Hello, future of AI fot Sciens! Yow ard you today?']


from IPython.display import clear_output


# model_type: word2vec, glove or fasttext
augment = naw.ContextualWordEmbsAug(model_path="bert-base-uncased", action="insert")
augmented_text = augment.augment(text)

clear_output()
print(f"Original:\n{text}")
print(f"Augmented Texts:\n{augmented_text}")

Original:
Hello, future of AI for Science! How are you today?
Augmented Texts:
['hello, future ceo of electronic ai for science! how important are you with today?']


!pip install -q sacremoses

     ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 897.5/897.5 kB 16.7 MB/s eta 0:00:00


back_translation_aug = naw.BackTranslationAug(
    from_model_name="facebook/wmt19-en-de", to_model_name="facebook/wmt19-de-en"
)
augmented_text = back_translation_aug.augment(text)

clear_output()
print(f"Original:\n{text}")
print(f"Augmented Texts:\n{augmented_text}")

Original:
Hello, future of AI for Science! How are you today?
Augmented Texts:
['Hello, the future of AI for science! How are you doing today?']


from torchtext.data.metrics import bleu_score

candidate_corpus = [["My", "full", "pytorch", "test"], ["Another", "Sentence"]]
references_corpus = [
    [["My", "full", "pytorch", "test"], ["Completely", "Different"]],
    [["No", "Match"]],
]
print(bleu_score(candidate_corpus, references_corpus))

0.8408964276313782


SOS_token = 0
EOS_token = 1


class Lang:
    def __init__(self, name):
        self.name = name
        self.word2index = {}
        self.word2count = {}
        self.index2word = {0: "SOS", 1: "EOS"}
        self.n_words = 2  # Count SOS and EOS

    def addSentence(self, sentence):
        for word in sentence.split(" "):
            self.addWord(word)

    def addWord(self, word):
        if word not in self.word2index:
            self.word2index[word] = self.n_words
            self.word2count[word] = 1
            self.index2word[self.n_words] = word
            self.n_words += 1
        else:
            self.word2count[word] += 1


import re
import unicodedata


def unicodeToAscii(s):
    return "".join(
        c for c in unicodedata.normalize("NFD", s) if unicodedata.category(c) != "Mn"
    )


def normalizeString(s):
    s = unicodeToAscii(s.lower().strip())
    s = re.sub(r"([.!?])", r" \1", s)
    s = re.sub(r"[^a-zA-Z!?]+", r" ", s)
    return s.strip()


def normalizeStringRu(s):
    s = unicodeToAscii(s.lower().strip())
    s = re.sub(r"([.!?])", r" \1", s)
    s = re.sub(r"[^a-яА-Я!?]+", r" ", s)
    return s.strip()


import torch
import random
import numpy as np


def set_random_seed(seed):
    torch.manual_seed(seed)
    torch.cuda.manual_seed(seed)
    np.random.seed(seed)
    random.seed(seed)


set_random_seed(42)


# Source: https://www.manythings.org/anki/
!wget -q https://edunet.kea.su/repo/EduNet-web_dependencies/datasets/eng_rus_vocab.txt


from io import open


def readLangs(lang1, lang2, reverse=False):
    print("Reading lines...")

    # Read the file and split into lines
    lines = (
        open("%s_%s_vocab.txt" % (lang1, lang2), encoding="utf-8")
        .read()
        .strip()
        .split("\n")
    )

    # Split every line into pairs and normalize
    pairs = [l.split("\t")[:2] for l in lines]
    eng = [normalizeString(s[0]) for s in pairs]
    rus = [normalizeStringRu(s[1]) for s in pairs]
    pairs = list(zip(rus, eng))

    # Reverse pairs, make Lang instances
    if reverse:
        pairs = [list(reversed(p)) for p in pairs]
        input_lang = Lang(lang2)
        output_lang = Lang(lang1)
    else:
        input_lang = Lang(lang1)
        output_lang = Lang(lang2)

    return input_lang, output_lang, pairs


max_length = 10

eng_prefixes = (
    "i am ",
    "i m ",
    "he is",
    "he s ",
    "she is",
    "she s ",
    "you are",
    "you re ",
    "we are",
    "we re ",
    "they are",
    "they re ",
)


def filterPair(p):
    return (
        len(p[0].split(" ")) < max_length
        and len(p[1].split(" ")) < max_length
        and p[1].startswith(eng_prefixes)
    )


def filterPairs(pairs):
    return [pair for pair in pairs if filterPair(pair)]


import random


def prepareData(lang1, lang2, reverse=False):
    input_lang, output_lang, pairs = readLangs(lang1, lang2, reverse)
    print("Read %s sentence pairs" % len(pairs))
    pairs = filterPairs(pairs)
    print("Trimmed to %s sentence pairs" % len(pairs))
    print("Counting words...")
    for pair in pairs:
        input_lang.addSentence(pair[0])
        output_lang.addSentence(pair[1])
    print("Counted words:")
    print(input_lang.name, input_lang.n_words)
    print(output_lang.name, output_lang.n_words)
    return input_lang, output_lang, pairs


input_lang, output_lang, pairs = prepareData("eng", "rus", False)
print(random.choice(pairs))

Reading lines...
Read 496059 sentence pairs
Trimmed to 30724 sentence pairs
Counting words...
Counted words:
eng 10510
rus 4349
('простите что была так груба', 'i m sorry that i was so rude')


import torch


device = torch.device("cuda" if torch.cuda.is_available() else "cpu")


def indexesFromSentence(lang, sentence):
    return [lang.word2index[word] for word in sentence.split(" ")]


def tensorFromSentence(lang, sentence):
    indexes = indexesFromSentence(lang, sentence)
    indexes.append(EOS_token)
    return torch.tensor(indexes, dtype=torch.long, device=device).view(1, -1)


def tensorsFromPair(pair):
    input_tensor = tensorFromSentence(input_lang, pair[0])
    target_tensor = tensorFromSentence(output_lang, pair[1])
    return (input_tensor, target_tensor)


from torch import nn


class EncoderRNN(nn.Module):
    def __init__(self, input_size, hidden_size, dropout_p=0.1):
        super(EncoderRNN, self).__init__()
        self.hidden_size = hidden_size

        self.embedding = nn.Embedding(input_size, hidden_size)
        self.gru = nn.GRU(hidden_size, hidden_size, batch_first=True)
        self.dropout = nn.Dropout(dropout_p)

    def forward(self, input):
        embedded = self.dropout(self.embedding(input))
        output, hidden = self.gru(embedded)
        return output, hidden


import torch.nn.functional as F


class DecoderRNN(nn.Module):
    def __init__(self, hidden_size, output_size):
        super(DecoderRNN, self).__init__()
        self.embedding = nn.Embedding(output_size, hidden_size)
        self.gru = nn.GRU(hidden_size, hidden_size, batch_first=True)
        self.out = nn.Linear(hidden_size, output_size)

    def forward(self, encoder_outputs, encoder_hidden, target_tensor=None):
        batch_size = encoder_outputs.size(0)
        decoder_input = torch.empty(
            batch_size, 1, dtype=torch.long, device=device
        ).fill_(SOS_token)
        decoder_hidden = encoder_hidden
        decoder_outputs = []

        for i in range(max_length):
            decoder_output, decoder_hidden = self.forward_step(
                decoder_input, decoder_hidden, encoder_outputs
            )
            decoder_outputs.append(decoder_output)

            if target_tensor is not None:
                # Teacher forcing: Feed the target as the next input
                decoder_input = target_tensor[:, i].unsqueeze(1)  # Teacher forcing
            else:
                # Without teacher forcing: use its own predictions as the next input
                _, topi = decoder_output.topk(1)
                decoder_input = topi.squeeze(
                    -1
                ).detach()  # detach from history as input

        decoder_outputs = torch.cat(decoder_outputs, dim=1)
        decoder_outputs = F.log_softmax(decoder_outputs, dim=-1)
        return (
            decoder_outputs,
            decoder_hidden,
            None,
        )  # We return `None` for consistency in the training loop

    def forward_step(self, input, hidden, encoder_outputs):
        output = self.embedding(input)
        output = F.relu(output)
        output, hidden = self.gru(output, hidden)
        output = self.out(output)
        return output, hidden


import numpy as np
from torch.utils.data import TensorDataset, DataLoader, RandomSampler


def get_dataloader(batch_size):
    input_lang, output_lang, pairs = prepareData("eng", "rus", False)

    n = len(pairs)
    input_ids = np.zeros((n, max_length), dtype=np.int32)
    target_ids = np.zeros((n, max_length), dtype=np.int32)

    for idx, (inp, tgt) in enumerate(pairs):
        inp_ids = indexesFromSentence(input_lang, inp)
        tgt_ids = indexesFromSentence(output_lang, tgt)
        inp_ids.append(EOS_token)
        tgt_ids.append(EOS_token)
        input_ids[idx, : len(inp_ids)] = inp_ids
        target_ids[idx, : len(tgt_ids)] = tgt_ids

    train_data = TensorDataset(
        torch.LongTensor(input_ids).to(device), torch.LongTensor(target_ids).to(device)
    )

    train_sampler = RandomSampler(train_data)
    train_dataloader = DataLoader(
        train_data, sampler=train_sampler, batch_size=batch_size
    )
    return input_lang, output_lang, train_dataloader


def train_epoch(
    dataloader, encoder, decoder, encoder_optimizer, decoder_optimizer, criterion
):
    total_loss = 0
    for data in dataloader:
        input_tensor, target_tensor = data

        encoder_optimizer.zero_grad()
        decoder_optimizer.zero_grad()

        encoder_outputs, encoder_hidden = encoder(input_tensor)
        decoder_outputs, _, _ = decoder(encoder_outputs, encoder_hidden, target_tensor)

        loss = criterion(
            decoder_outputs.view(-1, decoder_outputs.size(-1)), target_tensor.view(-1)
        )
        loss.backward()

        encoder_optimizer.step()
        decoder_optimizer.step()

        total_loss += loss.item()

    return total_loss / len(dataloader)


import time
import math


def asMinutes(s):
    m = math.floor(s / 60)
    s -= m * 60
    return "%dm %ds" % (m, s)


def timeSince(since, percent):
    now = time.time()
    s = now - since
    es = s / (percent)
    rs = es - s
    return "%s (- %s)" % (asMinutes(s), asMinutes(rs))


from torch import optim


def train(
    train_dataloader,
    encoder,
    decoder,
    num_epochs,
    learning_rate=0.001,
    print_every=100,
    plot_every=100,
):
    start = time.time()
    plot_losses = []
    print_loss_total = 0  # Reset every print_every
    plot_loss_total = 0  # Reset every plot_every

    encoder_optimizer = optim.Adam(encoder.parameters(), lr=learning_rate)
    decoder_optimizer = optim.Adam(decoder.parameters(), lr=learning_rate)
    criterion = nn.NLLLoss()

    for epoch in range(1, num_epochs + 1):
        loss = train_epoch(
            train_dataloader,
            encoder,
            decoder,
            encoder_optimizer,
            decoder_optimizer,
            criterion,
        )
        print_loss_total += loss
        plot_loss_total += loss

        if epoch % print_every == 0:
            print_loss_avg = print_loss_total / print_every
            print_loss_total = 0
            print(
                "%s (%d %d%%) %.4f"
                % (
                    timeSince(start, epoch / num_epochs),
                    epoch,
                    epoch / num_epochs * 100,
                    print_loss_avg,
                )
            )

        if epoch % plot_every == 0:
            plot_loss_avg = plot_loss_total / plot_every
            plot_losses.append(plot_loss_avg)
            plot_loss_total = 0

    showPlot(plot_losses)


import matplotlib.pyplot as plt
import matplotlib.ticker as ticker

%matplotlib inline
plt.switch_backend("agg")


def showPlot(points):
    plt.figure()
    fig, ax = plt.subplots()
    # this locator puts ticks at regular intervals
    loc = ticker.MultipleLocator(base=0.2)
    ax.yaxis.set_major_locator(loc)
    plt.plot(points)
    plt.show()


hidden_size = 128
batch_size = 32

input_lang, output_lang, train_dataloader = get_dataloader(batch_size)

encoder = EncoderRNN(input_lang.n_words, hidden_size).to(device)
decoder = DecoderRNN(hidden_size, output_lang.n_words).to(device)

# train(train_dataloader, encoder, decoder, num_epochs=80, print_every=5, plot_every=5)

Reading lines...
Read 496059 sentence pairs
Trimmed to 30724 sentence pairs
Counting words...
Counted words:
eng 10510
rus 4349


!wget -q https://edunet.kea.su/repo/EduNet-content/dev-2.0/L09/weights/encoder_weights.pth
!wget -q https://edunet.kea.su/repo/EduNet-content/dev-2.0/L09/weights/decoder_weights.pth


encoder.load_state_dict(torch.load("/content/encoder_weights.pth", map_location=device))
decoder.load_state_dict(torch.load("/content/decoder_weights.pth", map_location=device))

<All keys matched successfully>


def evaluate(encoder, decoder, sentence, input_lang, output_lang):
    with torch.no_grad():
        input_tensor = tensorFromSentence(input_lang, sentence)

        encoder_outputs, encoder_hidden = encoder(input_tensor)
        decoder_outputs, decoder_hidden, _ = decoder(encoder_outputs, encoder_hidden)

        _, topi = decoder_outputs.topk(1)
        decoded_ids = topi.squeeze()
        decoded_words = []
        for idx in decoded_ids:
            if idx.item() == EOS_token:
                decoded_words.append("<EOS>")
                break
            decoded_words.append(output_lang.index2word[idx.item()])
    return decoded_words


def evaluateRandomly(encoder, decoder, n=10):
    for i in range(n):
        pair = random.choice(pairs)
        print("RUS", pair[0])
        print("ENG", pair[1])
        output_words = evaluate(encoder, decoder, pair[0], input_lang, output_lang)
        output_sentence = " ".join(output_words)
        print("DNN", output_sentence)
        print("")


encoder.eval()
decoder.eval()
evaluateRandomly(encoder, decoder)

RUS я не совсем уверен
ENG i m not so sure
DNN i m not totally sure are work <EOS>

RUS я доволен
ENG i m pleased
DNN i m satisfied with the dog <EOS>

RUS я никуда с тобои не поеду
ENG i m not going anywhere with you
DNN i m not going anywhere with you <EOS>

RUS я привык ждать
ENG i m used to waiting
DNN i m used to doing that <EOS>

RUS вы оба симпатичные
ENG you re both pretty
DNN you re both pretty young <EOS>

RUS я всего лишь учитель
ENG i m just a teacher
DNN i m just a teacher <EOS>

RUS я переводчица
ENG i m a translator
DNN i m a member of the team <EOS>

RUS в настоящии момент я работаю в бостоне
ENG i m currently working in boston
DNN i m currently working in boston <EOS>

RUS я смертельно устал
ENG i am dead tired
DNN i am very tired from swimming <EOS>

RUS уверен что том тебя не ненавидит
ENG i m sure tom doesn t hate you
DNN i m sure tom doesn t hate you <EOS>

	Month	Passengers
0	1949-01	112
1	1949-02	118
2	1949-03	132
3	1949-04	129
4	1949-05	121

Особенности данных¶

Теория и классические подходы¶

Статистические модели предсказания¶

Разделение данных¶

Рекуррентные нейронные сети¶

Области применения¶

Основная идея¶

Базовый RNN блок¶

RNNCell¶

RNN блок в PyTorch¶

Слои (Stacked RNNs)¶

Пример прогнозирования временного ряда¶

Нейросетевой подход¶

Шкалирование данных¶

Формирование ансамблей данных¶

Разобьем на train и test¶

Создание и обучение модели¶

Обучение¶

Тестирование¶

Проблемы RNN¶

LSTM¶

LSTMCell¶

LSTM в PyTorch¶

Пример¶

Bidirectional¶

Скрытое состояние¶

Модификации LSTM¶

Типы задач¶

Пример посимвольной генерации текста¶

Подготовка данных¶

Выравнивание данных (Padding)¶

Разбиение данных¶

Кодирование¶

One-hot encoding¶

Создание и обучение модели¶

Обучение¶

Тестирование¶

Представление данных¶

Токенизация¶

TF-IDF¶

Word2Vec¶

Слой эмбеддингов¶

Размер словаря¶

Byte Pair Encoding¶

Аугментация¶

Аудио¶

Background Noise¶

Time Stretch¶

Pitch Shift¶

Совмещаем несколько аугментаций вместе¶

Текст¶

Аугментация символов¶

Аугментация слов¶

Аугментация предложений¶

NLP метрики¶

Задача Sequence-to-Sequence¶

Реализация¶

Загрузка и предобработка данных¶

Кодировщик-декодировщик¶

Обучение¶