Chapter 1: Perceptron Basics

Learning Objectives

By reading this chapter, you will be able to:

• ✅ Understand the structure and operating principles of the perceptron
• ✅ Explain the roles of weights and biases
• ✅ Implement logic gates (AND, OR, NAND) in Python
• ✅ Understand the concept of linear separability
• ✅ Learn why multilayer networks are needed from the XOR problem

1.1 What is a Perceptron

Historical Background

The perceptron was devised by Frank Rosenblatt in 1957. It is the first machine learning algorithm that mimics the nerve cells (neurons) of the human brain.

"A perceptron receives multiple signals as input and produces a single output signal. It multiplies the input signals by weights, sums them up, and fires (outputs 1) when the sum exceeds a threshold."

Structure of the Perceptron

Mathematical Expression:

$$ y = \begin{cases} 1 & \text{if } w_1x_1 + w_2x_2 + b > 0 \\ 0 & \text{otherwise} \end{cases} $$

Or, using a step function:

$$ y = h(w_1x_1 + w_2x_2 + b) $$

where $h(x)$ is the Heaviside function:

$$ h(x) = \begin{cases} 1 & \text{if } x > 0 \\ 0 & \text{otherwise} \end{cases} $$

Description of Components

1.2 Implementing Logic Gates

AND Gate

Truth Table:

Python Implementation:

import numpy as np

def AND(x1, x2):
    """
    AND gate implementation
    Weights: w1=0.5, w2=0.5
    Bias: b=-0.7
    """
    x = np.array([x1, x2])
    w = np.array([0.5, 0.5])
    b = -0.7

    # Compute the weighted sum
    tmp = np.sum(w * x) + b

    # Activation function (step function)
    if tmp > 0:
        return 1
    else:
        return 0

# Test
print("=== AND Gate ===")
print(f"AND(0, 0) = {AND(0, 0)}")  # 0
print(f"AND(0, 1) = {AND(0, 1)}")  # 0
print(f"AND(1, 0) = {AND(1, 0)}")  # 0
print(f"AND(1, 1) = {AND(1, 1)}")  # 1

Output:

=== AND Gate ===
AND(0, 0) = 0
AND(0, 1) = 0
AND(1, 0) = 0
AND(1, 1) = 1

OR Gate

Truth Table:

def OR(x1, x2):
    """
    OR gate implementation
    Weights: w1=0.5, w2=0.5
    Bias: b=-0.2
    """
    x = np.array([x1, x2])
    w = np.array([0.5, 0.5])
    b = -0.2  # Fires more easily than AND

    tmp = np.sum(w * x) + b

    if tmp > 0:
        return 1
    else:
        return 0

# Test
print("\n=== OR Gate ===")
print(f"OR(0, 0) = {OR(0, 0)}")  # 0
print(f"OR(0, 1) = {OR(0, 1)}")  # 1
print(f"OR(1, 0) = {OR(1, 0)}")  # 1
print(f"OR(1, 1) = {OR(1, 1)}")  # 1

NAND Gate

NAND (NOT AND) inverts the output of AND.

def NAND(x1, x2):
    """
    NAND gate implementation
    Weights: w1=-0.5, w2=-0.5 (negative weights)
    Bias: b=0.7
    """
    x = np.array([x1, x2])
    w = np.array([-0.5, -0.5])  # Negative weights
    b = 0.7

    tmp = np.sum(w * x) + b

    if tmp > 0:
        return 1
    else:
        return 0

# Test
print("\n=== NAND Gate ===")
print(f"NAND(0, 0) = {NAND(0, 0)}")  # 1
print(f"NAND(0, 1) = {NAND(0, 1)}")  # 1
print(f"NAND(1, 0) = {NAND(1, 0)}")  # 1
print(f"NAND(1, 1) = {NAND(1, 1)}")  # 0

General-Purpose Perceptron Class

class Perceptron:
    """General-purpose perceptron class"""

    def __init__(self, weights, bias):
        """
        Args:
            weights: numpy array of weights
            bias: bias value
        """
        self.w = np.array(weights)
        self.b = bias

    def forward(self, x):
        """
        Forward propagation

        Args:
            x: array of input values

        Returns:
            0 or 1
        """
        tmp = np.sum(self.w * x) + self.b
        return 1 if tmp > 0 else 0

    def __call__(self, *inputs):
        """Make the instance callable"""
        x = np.array(inputs)
        return self.forward(x)

# Define logic gates using perceptrons
and_gate = Perceptron(weights=[0.5, 0.5], bias=-0.7)
or_gate = Perceptron(weights=[0.5, 0.5], bias=-0.2)
nand_gate = Perceptron(weights=[-0.5, -0.5], bias=0.7)

# Test
print("\n=== General-Purpose Perceptron ===")
print(f"AND(1, 1) = {and_gate(1, 1)}")    # 1
print(f"OR(0, 1) = {or_gate(0, 1)}")      # 1
print(f"NAND(1, 1) = {nand_gate(1, 1)}")  # 0

1.3 The Roles of Weights and Biases

The Meaning of Weights

A weight represents the importance of an input.

• Large weight: The input is important
• Small weight: The input is not important
• Negative weight: The input suppresses the output

import matplotlib.pyplot as plt

# Output as the weight is varied
def visualize_weight_effect():
    """Visualize the effect of the weight"""
    weights = np.linspace(-2, 2, 100)
    x1, x2 = 1, 1
    b = -0.7

    outputs = []
    for w in weights:
        tmp = w * x1 + w * x2 + b
        y = 1 if tmp > 0 else 0
        outputs.append(y)

    plt.figure(figsize=(10, 4))
    plt.plot(weights, outputs, linewidth=2)
    plt.xlabel('Weight (w1 = w2 = w)', fontsize=12)
    plt.ylabel('Output', fontsize=12)
    plt.title('Weight Variation and Perceptron Output (x1=1, x2=1, b=-0.7)', fontsize=14)
    plt.grid(True, alpha=0.3)
    plt.ylim(-0.1, 1.1)
    plt.axhline(y=0.5, color='r', linestyle='--', alpha=0.5)
    plt.axvline(x=0.35, color='g', linestyle='--', alpha=0.5, label='Threshold')
    plt.legend()
    plt.show()

visualize_weight_effect()

The Meaning of Bias

The bias adjusts how easily the perceptron fires.

• Large bias: Fires easily (output tends toward 1)
• Small bias: Fires less easily (output tends toward 0)

def compare_bias():
    """Compare different bias values"""
    x1, x2 = 0.5, 0.5
    w1, w2 = 0.5, 0.5

    biases = [-1.0, -0.5, 0.0, 0.5, 1.0]

    print("=== Effect of Bias ===")
    print(f"Inputs: x1={x1}, x2={x2}")
    print(f"Weights: w1={w1}, w2={w2}")
    print()

    for b in biases:
        tmp = w1*x1 + w2*x2 + b
        y = 1 if tmp > 0 else 0
        print(f"Bias b={b:5.1f} → Sum={tmp:5.2f} → Output={y}")

compare_bias()

Output:

=== Effect of Bias ===
Inputs: x1=0.5, x2=0.5
Weights: w1=0.5, w2=0.5

Bias b= -1.0 → Sum=-0.50 → Output=0
Bias b= -0.5 → Sum= 0.00 → Output=0
Bias b=  0.0 → Sum= 0.50 → Output=1
Bias b=  0.5 → Sum= 1.00 → Output=1
Bias b=  1.0 → Sum= 1.50 → Output=1

1.4 Linear Separability

Explanation of the Concept

Linearly separable means that the data can be separated by a single straight line (in 2D) or a plane (in higher dimensions).

A perceptron can only learn linearly separable problems.

Decision Boundary of the AND Gate

import matplotlib.pyplot as plt
import numpy as np

def plot_decision_boundary_AND():
    """Visualize the decision boundary of the AND gate"""
    # Data points
    x1 = np.array([0, 0, 1, 1])
    x2 = np.array([0, 1, 0, 1])
    y = np.array([0, 0, 0, 1])  # AND outputs

    # Plot
    plt.figure(figsize=(8, 6))

    # Class 0 (output 0)
    plt.scatter(x1[y==0], x2[y==0], s=200, c='blue', marker='o',
                label='Output = 0', edgecolors='k', linewidths=2)

    # Class 1 (output 1)
    plt.scatter(x1[y==1], x2[y==1], s=200, c='red', marker='s',
                label='Output = 1', edgecolors='k', linewidths=2)

    # Decision boundary: w1*x1 + w2*x2 + b = 0
    # 0.5*x1 + 0.5*x2 - 0.7 = 0
    # x2 = -x1 + 1.4
    x_line = np.linspace(-0.5, 1.5, 100)
    y_line = -x_line + 1.4
    plt.plot(x_line, y_line, 'g--', linewidth=2, label='Decision boundary')

    # Fill the regions
    plt.fill_between(x_line, y_line, 2, alpha=0.2, color='red', label='Output=1 region')
    plt.fill_between(x_line, -1, y_line, alpha=0.2, color='blue', label='Output=0 region')

    plt.xlim(-0.5, 1.5)
    plt.ylim(-0.5, 1.5)
    plt.xlabel('x1', fontsize=14)
    plt.ylabel('x2', fontsize=14)
    plt.title('Decision Boundary of the AND Gate', fontsize=16, fontweight='bold')
    plt.grid(True, alpha=0.3)
    plt.legend(fontsize=10, loc='upper right')
    plt.show()

plot_decision_boundary_AND()

Decision Boundary of the OR Gate

def plot_decision_boundary_OR():
    """Visualize the decision boundary of the OR gate"""
    # Data points
    x1 = np.array([0, 0, 1, 1])
    x2 = np.array([0, 1, 0, 1])
    y = np.array([0, 1, 1, 1])  # OR outputs

    plt.figure(figsize=(8, 6))

    # Class 0
    plt.scatter(x1[y==0], x2[y==0], s=200, c='blue', marker='o',
                label='Output = 0', edgecolors='k', linewidths=2)

    # Class 1
    plt.scatter(x1[y==1], x2[y==1], s=200, c='red', marker='s',
                label='Output = 1', edgecolors='k', linewidths=2)

    # Decision boundary: 0.5*x1 + 0.5*x2 - 0.2 = 0
    # x2 = -x1 + 0.4
    x_line = np.linspace(-0.5, 1.5, 100)
    y_line = -x_line + 0.4
    plt.plot(x_line, y_line, 'g--', linewidth=2, label='Decision boundary')

    plt.fill_between(x_line, y_line, 2, alpha=0.2, color='red')
    plt.fill_between(x_line, -1, y_line, alpha=0.2, color='blue')

    plt.xlim(-0.5, 1.5)
    plt.ylim(-0.5, 1.5)
    plt.xlabel('x1', fontsize=14)
    plt.ylabel('x2', fontsize=14)
    plt.title('Decision Boundary of the OR Gate', fontsize=16, fontweight='bold')
    plt.grid(True, alpha=0.3)
    plt.legend(fontsize=10, loc='upper right')
    plt.show()

plot_decision_boundary_OR()

1.5 The XOR Problem - Limitations of the Perceptron

What is the XOR Gate

XOR (Exclusive OR) is a logical operation that "outputs 1 only when exactly one of the inputs is 1".

A Single-Layer Perceptron Cannot Realize It

def plot_XOR_problem():
    """Visualize the XOR problem - not linearly separable"""
    x1 = np.array([0, 0, 1, 1])
    x2 = np.array([0, 1, 0, 1])
    y = np.array([0, 1, 1, 0])  # XOR outputs

    plt.figure(figsize=(8, 6))

    # Class 0
    plt.scatter(x1[y==0], x2[y==0], s=200, c='blue', marker='o',
                label='Output = 0', edgecolors='k', linewidths=2)

    # Class 1
    plt.scatter(x1[y==1], x2[y==1], s=200, c='red', marker='s',
                label='Output = 1', edgecolors='k', linewidths=2)

    # Show that it is not linearly separable
    plt.text(0.5, 1.3, 'Cannot be separated\nby a single line!',
             fontsize=14, ha='center',
             bbox=dict(boxstyle='round', facecolor='yellow', alpha=0.5))

    plt.xlim(-0.5, 1.5)
    plt.ylim(-0.5, 1.5)
    plt.xlabel('x1', fontsize=14)
    plt.ylabel('x2', fontsize=14)
    plt.title('XOR Problem - Not Linearly Separable', fontsize=16, fontweight='bold')
    plt.grid(True, alpha=0.3)
    plt.legend(fontsize=10, loc='upper right')
    plt.show()

plot_XOR_problem()

Solving It with a Multilayer Perceptron

The XOR problem can be solved by combining multiple perceptrons.

def XOR(x1, x2):
    """
    XOR gate implementation
    Combines NAND, OR, and AND
    """
    s1 = NAND(x1, x2)
    s2 = OR(x1, x2)
    y = AND(s1, s2)
    return y

# Test
print("\n=== XOR Gate (Multilayer Perceptron) ===")
print(f"XOR(0, 0) = {XOR(0, 0)}")  # 0
print(f"XOR(0, 1) = {XOR(0, 1)}")  # 1
print(f"XOR(1, 0) = {XOR(1, 0)}")  # 1
print(f"XOR(1, 1) = {XOR(1, 1)}")  # 0

Output:

=== XOR Gate (Multilayer Perceptron) ===
XOR(0, 0) = 0
XOR(0, 1) = 1
XOR(1, 0) = 1
XOR(1, 1) = 0

Increased Expressiveness Through Multiple Layers

Key Insight: By stacking perceptrons into multiple layers, problems that are not linearly separable can also be solved. This is the essence of neural networks.

1.6 Chapter Summary

What We Learned

1. Structure of the Perceptron

   • Inputs, weights, bias, activation function, output
   • Formula: $y = h(w_1x_1 + w_2x_2 + b)$

2. Implementing Logic Gates

   • AND, OR, and NAND can be realized with a single-layer perceptron
   • Implemented by setting appropriate weights and biases

3. Roles of Weights and Biases

   • Weight: importance of an input
   • Bias: ease of firing

4. Linear Separability

   • A perceptron can only solve linearly separable problems
   • The decision boundary is a line (2D) or a hyperplane (higher dimensions)

5. XOR Problem and Multiple Layers

   • XOR is not linearly separable
   • Solvable with a multilayer perceptron
   • This is the path to deep learning

Key Points

To the Next Chapter

In Chapter 2, we will learn about the multilayer perceptron (MLP) and backpropagation:

• Structure of multilayer networks
• Backpropagation
• Learning with gradient descent
• Complete implementation with NumPy

Exercises

Exercise 1 (Difficulty: easy)

Determine whether each of the following statements is true or false.

1. A perceptron has weights and a bias
2. The AND gate can be implemented with a single-layer perceptron
3. The XOR gate can be implemented with a single-layer perceptron
4. The larger the bias, the harder it is for the perceptron to fire

Exercise 2 (Difficulty: medium)

Implement a NOR gate (NOT OR). The truth table is as follows:

def NOR(x1, x2):
    """
    NOR gate implementation
    Weights: w1=-0.5, w2=-0.5
    Bias: b=0.2
    """
    x = np.array([x1, x2])
    w = np.array([-0.5, -0.5])
    b = 0.2

    tmp = np.sum(w * x) + b

    if tmp > 0:
        return 1
    else:
        return 0

# Test
print("=== NOR Gate ===")
for x1, x2 in [(0,0), (0,1), (1,0), (1,1)]:
    y = NOR(x1, x2)
    print(f"NOR({x1}, {x2}) = {y}")

=== NOR Gate ===
NOR(0, 0) = 1
NOR(0, 1) = 0
NOR(1, 0) = 0
NOR(1, 1) = 0

Exercise 3 (Difficulty: medium)

Implement a 3-input AND gate. The output should be 1 only when all three inputs are 1.

def AND3(x1, x2, x3):
    """
    3-input AND gate implementation
    Weights: w1=0.5, w2=0.5, w3=0.5
    Bias: b=-1.2
    """
    x = np.array([x1, x2, x3])
    w = np.array([0.5, 0.5, 0.5])
    b = -1.2  # Fires only when the sum of the three inputs reaches 1.5

    tmp = np.sum(w * x) + b

    if tmp > 0:
        return 1
    else:
        return 0

# Test
print("=== 3-Input AND Gate ===")
for x1 in [0, 1]:
    for x2 in [0, 1]:
        for x3 in [0, 1]:
            y = AND3(x1, x2, x3)
            print(f"AND3({x1}, {x2}, {x3}) = {y}")

=== 3-Input AND Gate ===
AND3(0, 0, 0) = 0
AND3(0, 0, 1) = 0
AND3(0, 1, 0) = 0
AND3(0, 1, 1) = 0
AND3(1, 0, 0) = 0
AND3(1, 0, 1) = 0
AND3(1, 1, 0) = 0
AND3(1, 1, 1) = 1

Exercise 4 (Difficulty: hard)

Implement an XNOR gate (the negation of XOR) using a multilayer perceptron. Truth table:

def XNOR_v1(x1, x2):
    """
    XNOR gate (method 1): invert the output of XOR
    """
    xor_out = XOR(x1, x2)
    # Inversion requires a NOT gate (realizable with NAND)
    return 1 if xor_out == 0 else 0

def XNOR_v2(x1, x2):
    """
    XNOR gate (method 2): combination of OR, NAND, and NAND
    """
    s1 = OR(x1, x2)
    s2 = NAND(x1, x2)
    y = NAND(s1, s2)
    return y

# Test
print("=== XNOR Gate ===")
print("Method 1 (XOR + NOT):")
for x1, x2 in [(0,0), (0,1), (1,0), (1,1)]:
    y = XNOR_v1(x1, x2)
    print(f"XNOR({x1}, {x2}) = {y}")

print("\nMethod 2 (OR + NAND + NAND):")
for x1, x2 in [(0,0), (0,1), (1,0), (1,1)]:
    y = XNOR_v2(x1, x2)
    print(f"XNOR({x1}, {x2}) = {y}")

Exercise 5 (Difficulty: hard)

Solve a simple classification problem using a perceptron. Find weights and a bias that correctly classify the following data:

• Class 0: (0, 0), (0, 1)
• Class 1: (1, 0), (1, 1)

def custom_classifier(x1, x2):
    """
    Custom classifier
    x1 is the important feature
    """
    w1 = 1.0  # Emphasize x1
    w2 = 0.0  # Ignore x2
    b = -0.5

    tmp = w1*x1 + w2*x2 + b
    return 1 if tmp > 0 else 0

# Test
print("=== Custom Classifier ===")
data = [
    ((0, 0), 0),
    ((0, 1), 0),
    ((1, 0), 1),
    ((1, 1), 1)
]

correct = 0
for (x1, x2), expected in data:
    pred = custom_classifier(x1, x2)
    result = "✓" if pred == expected else "✗"
    print(f"Input({x1}, {x2}) → Predicted={pred}, Expected={expected} {result}")
    if pred == expected:
        correct += 1

print(f"\nAccuracy: {correct}/{len(data)} = {100*correct/len(data):.1f}%")

References

1. Rosenblatt, F. (1958). "The Perceptron: A Probabilistic Model for Information Storage and Organization in the Brain." Psychological Review, 65(6), 386-408.
2. Minsky, M., & Papert, S. (1969). Perceptrons: An Introduction to Computational Geometry. MIT Press.
3. Saito, K. (2016). Deep Learning from Scratch. O'Reilly Japan.