51. Iterators & Generators: Lazy Evaluation in JavaScript

Introduction

An iterator is an object that defines a way to traverse a data collection one item at a time. A generator is a special function that can pause and resume its execution, producing a sequence of values lazily. Together, they enable the lazy evaluation pattern where data is processed only when needed.

For IT students working with large datasets such as the data of an entire student cohort, iterators and generators are very efficient because you don't need to load all the data into memory at once.

Why Are Iterators and Generators Important?

When working with large datasets (thousands of student records, class schedules, grade transcripts), loading everything into memory at once can cause crashes or slowness. Generators let us process data one item at a time only when it is needed (lazy evaluation), saving memory and time.

Iterator Protocol

// An iterator must have a next() method that returns { value, done }

// Manual iterator for an array
function createArrayIterator(arr) {
    let index = 0;

    return {
        next() {
            if (index < arr.length) {
                return { value: arr[index++], done: false };
            }
            return { value: undefined, done: true };
        }
    };
}

const arr = [10, 20, 30];
const iter = createArrayIterator(arr);

console.log(iter.next()); // { value: 10, done: false }
console.log(iter.next()); // { value: 20, done: false }
console.log(iter.next()); // { value: 30, done: false }
console.log(iter.next()); // { value: undefined, done: true }

Symbol.iterator

// Make an object iterable with for...of
class Range {
    constructor(start, end, step = 1) {
        this.start = start;
        this.end = end;
        this.step = step;
    }

    [Symbol.iterator]() {
        let current = this.start;
        const end = this.end;
        const step = this.step;

        return {
            next() {
                if (current <= end) {
                    const value = current;
                    current += step;
                    return { value, done: false };
                }
                return { value: undefined, done: true };
            }
        };
    }

    toArray() {
        return [...this];
    }
}

// Usage
const range = new Range(1, 10);
for (const num of range) {
    console.log(num); // 1, 2, 3, ..., 10
}

const evenRange = new Range(2, 20, 2);
console.log([...evenRange]); // [2, 4, 6, 8, 10, 12, 14, 16, 18, 20]
console.log(range.toArray()); // [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]

// Iterator for tree traversal
class TreeNode {
    constructor(value, children = []) {
        this.value = value;
        this.children = children;
    }

    // Depth-first iterator
    [Symbol.iterator]() {
        const stack = [this];

        return {
            next() {
                while (stack.length > 0) {
                    const node = stack.pop();
                    // Push children in reverse for left-to-right traversal
                    for (let i = node.children.length - 1; i >= 0; i--) {
                        stack.push(node.children[i]);
                    }
                    return { value: node.value, done: false };
                }
                return { value: undefined, done: true };
            }
        };
    }
}

// Campus tree
const root = new TreeNode('University', [
    new TreeNode('Faculty of Engineering', [
        new TreeNode('Computer Science'),
        new TreeNode('Information Systems'),
        new TreeNode('Computer Engineering')
    ]),
    new TreeNode('Faculty of Science', [
        new TreeNode('Mathematics'),
        new TreeNode('Physics')
    ])
]);

for (const dept of root) {
    console.log(dept);
}
// University, Faculty of Engineering, Computer Science, Information Systems, Computer Engineering, Faculty of Science, Mathematics, Physics

Generator Functions

// Generators use function* syntax
function* countTo(n) {
    for (let i = 1; i <= n; i++) {
        yield i; // Pause and return the value
    }
}

// Usage
const counter = countTo(5);
console.log(counter.next()); // { value: 1, done: false }
console.log(counter.next()); // { value: 2, done: false }
console.log(counter.next()); // { value: 3, done: false }
console.log(counter.next()); // { value: 4, done: false }
console.log(counter.next()); // { value: 5, done: false }
console.log(counter.next()); // { value: undefined, done: true }

// Can be used in for...of
for (const num of countTo(5)) {
    console.log(num); // 1, 2, 3, 4, 5
}

// Generator with a parameter
function* paginate(items, pageSize) {
    let page = 0;

    while (page * pageSize < items.length) {
        const start = page * pageSize;
        const end = start + pageSize;
        yield {
            page: page + 1,
            data: items.slice(start, end),
            totalPages: Math.ceil(items.length / pageSize)
        };
        page++;
    }
}

// Usage
const allStudents = [
    { studentId: '001', name: 'Mike' },
    { studentId: '002', name: 'John' },
    { studentId: '003', name: 'Anna' },
    { studentId: '004', name: 'Diana' },
    { studentId: '005', name: 'Eric' },
    { studentId: '006', name: 'Frank' }
];

const pages = paginate(allStudents, 3);

for (const page of pages) {
    console.log(`Page ${page.page}/${page.totalPages}`);
    console.log(page.data);
}

yield and next() with Values

// Generators can receive values through next()
function* conversation() {
    const name = yield 'What is your name?';
    const age = yield `Hello ${name}, what is your age?`;
    yield `${name}, age ${age} years. Thank you!`;
}

const chat = conversation();
console.log(chat.next());          // { value: 'What is your name?', done: false }
console.log(chat.next('Mike'));    // { value: 'Hello Mike, what is your age?', done: false }
console.log(chat.next(20));        // { value: 'Mike, age 20 years. Thank you!', done: false }
console.log(chat.next());          // { value: undefined, done: true }

// Generator for a state machine
function* enrollmentProcess() {
    console.log('Starting registration...');
    const data = yield 'FILL_DATA';

    if (!data.studentId || !data.name) {
        return 'ERROR: Incomplete data';
    }

    const confirm = yield `Confirm: ${data.name} (${data.studentId}). Continue? (y/n)`;

    if (confirm !== 'y') {
        return 'Cancelled';
    }

    yield 'PAYMENT_PROCESS';

    const pay = yield 'Payment confirmation done? (y/n)';

    if (pay !== 'y') {
        return 'Awaiting payment';
    }

    return 'REGISTRATION_SUCCESS';
}

const process = enrollmentProcess();
let result = process.next();
console.log(result.value); // 'FILL_DATA'

result = process.next({ studentId: '1234567890', name: 'Mike' });
console.log(result.value); // 'Confirm: Mike (1234567890). Continue? (y/n)'

result = process.next('y');
console.log(result.value); // 'PAYMENT_PROCESS'

result = process.next();
console.log(result.value); // 'Payment confirmation done? (y/n)'

result = process.next('y');
console.log(result.value); // 'REGISTRATION_SUCCESS'

Delegation with yield*

// yield* delegates to another iterable
function* allCourses() {
    yield* semester1();
    yield* semester2();
    yield* semester3();
}

function* semester1() {
    yield 'Algorithms';
    yield 'Programming Dasar';
    yield 'Mathematics Diskrit';
}

function* semester2() {
    yield 'Web Programming';
    yield 'Database';
    yield 'Computer Networking';
}

function* semester3() {
    yield 'Artificial Intelligence';
    yield 'Software Engineering';
}

for (const course of allCourses()) {
    console.log(course);
}

// yield* with a recursive generator
function* flatten(arr) {
    for (const item of arr) {
        if (Array.isArray(item)) {
            yield* flatten(item); // Recursive delegation
        } else {
            yield item;
        }
    }
}

const nested = [1, [2, 3], [4, [5, 6]], 7];
console.log([...flatten(nested)]); // [1, 2, 3, 4, 5, 6, 7]

// Generator for merging multiple iterables
function* merge(...iterables) {
    for (const iterable of iterables) {
        yield* iterable;
    }
}

const a = [1, 2, 3];
const b = [4, 5, 6];
const c = [7, 8, 9];

console.log([...merge(a, b, c)]); // [1, 2, 3, 4, 5, 6, 7, 8, 9]

Async Iterators & Async Generators

// Async iterator protocol: next() returns a Promise
// Async generator: function* with async and await

async function* fetchStudents(page = 1, pageSize = 10) {
    let currentPage = page;

    while (true) {
        const response = await fetch(
            `/api/students?page=${currentPage}&size=${pageSize}`
        );
        const data = await response.json();

        if (data.students.length === 0) break;

        for (const student of data.students) {
            yield student;
        }

        if (!data.hasMore) break;
        currentPage++;
    }
}

// for await...of for async iterables
async function displayAllStudents() {
    for await (const student of fetchStudents()) {
        console.log(`${student.name} - ${student.studentId}`);
    }
}

// Async generator for streaming data
async function* readLargeFile(filePath) {
    const response = await fetch(`/api/files/${filePath}`);
    const reader = response.body.getReader();
    const decoder = new TextDecoder();
    let buffer = '';

    while (true) {
        const { done, value } = await reader.read();
        if (done) break;

        buffer += decoder.decode(value, { stream: true });
        const lines = buffer.split('\n');
        buffer = lines.pop(); // Keep the incomplete remainder

        for (const line of lines) {
            if (line.trim()) {
                yield JSON.parse(line);
            }
        }
    }

    // Process the remaining buffer
    if (buffer.trim()) {
        yield JSON.parse(buffer);
    }
}

// Usage
async function processFile() {
    for await (const record of readLargeFile('data-student.jsonl')) {
        console.log(`Processing: ${record.name}`);
    }
}

Real-World Example: Iterating a Student Dataset with Lazy Evaluation

// =====================================================
// Lazy Student Dataset Iterator
// =====================================================
class StudentDataset {
    #students;

    constructor(students) {
        this.#students = students;
    }

    // Lazy filter - doesn't create a new array
    *where(predicate) {
        for (const student of this.#students) {
            if (predicate(student)) {
                yield student;
            }
        }
    }

    // Lazy map - transform one at a time
    *select(transform) {
        for (const student of this.#students) {
            yield transform(student);
        }
    }

    // Lazy take - take the first n
    *take(n) {
        let count = 0;
        for (const student of this.#students) {
            if (count >= n) return;
            yield student;
            count++;
        }
    }

    // Lazy skip - skip the first n
    *skip(n) {
        let count = 0;
        for (const student of this.#students) {
            if (count >= n) {
                yield student;
            }
            count++;
        }
    }

    // Chunk - split into batches
    *chunk(size) {
        let chunk = [];
        for (const student of this.#students) {
            chunk.push(student);
            if (chunk.length === size) {
                yield chunk;
                chunk = [];
            }
        }
        if (chunk.length > 0) yield chunk;
    }

    // Aggregate operations
    count() {
        let n = 0;
        for (const _ of this.#students) n++;
        return n;
    }

    first() {
        for (const student of this.#students) {
            return student;
        }
        return null;
    }

    reduce(fn, initial) {
        let acc = initial;
        for (const student of this.#students) {
            acc = fn(acc, student);
        }
        return acc;
    }

    toArray() {
        return [...this.#students];
    }

    // Static factory from large data
    static fromGenerator(generator) {
        return new StudentDataset(generator);
    }
}

// Generator for student data from a "database"
function* studentDatabase(query = {}) {
    // Simulate incremental fetching from a database
    const allStudents = [
        { studentId: '001', name: 'Mike', majorCode: 'CS', gpa: 3.75, year: 2021 },
        { studentId: '002', name: 'John', majorCode: 'IS', gpa: 3.50, year: 2021 },
        { studentId: '003', name: 'Anna', majorCode: 'CS', gpa: 3.90, year: 2022 },
        { studentId: '004', name: 'Diana', majorCode: 'CE', gpa: 3.20, year: 2022 },
        { studentId: '005', name: 'Eric', majorCode: 'CS', gpa: 3.85, year: 2021 },
        { studentId: '006', name: 'Frank', majorCode: 'IS', gpa: 3.10, year: 2023 },
        { studentId: '007', name: 'Grace', majorCode: 'CS', gpa: 3.95, year: 2023 },
        { studentId: '008', name: 'Henry', majorCode: 'CE', gpa: 2.80, year: 2021 },
        { studentId: '009', name: 'Irene', majorCode: 'IS', gpa: 3.60, year: 2022 },
        { studentId: '010', name: 'Jack', majorCode: 'CS', gpa: 3.40, year: 2023 }
    ];

    for (const student of allStudents) {
        // Simulate a fetch delay
        // In a real case, this could yield from a database cursor
        if (query.majorCode && student.majorCode !== query.majorCode) continue;
        if (query.year && student.year !== query.year) continue;

        yield student;
    }
}

// =====================================================
// Usage
// =====================================================
const dataset = StudentDataset.fromGenerator(studentDatabase());

// Lazy pipeline - only process what is needed
const topTIStudents = dataset
    .where(s => s.majorCode === 'CS')
    .where(s => s.gpa >= 3.5)
    .select(s => ({
        name: s.name,
        studentId: s.studentId,
        gpa: s.gpa,
        status: s.gpa >= 3.8 ? 'Cum Laude' : 'Regular'
    }));

// Data is not processed until we iterate
for (const student of topTIStudents) {
    console.log(`${student.name} (${student.studentId}) - GPA: ${student.gpa} [${student.status}]`);
}

// Another example: lazy pagination
const allStudents = StudentDataset.fromGenerator(studentDatabase());

let pageNum = 1;
for (const page of allStudents.chunk(3)) {
    console.log(`\n=== Page ${pageNum} ===`);
    page.forEach(s => console.log(`${s.name} - ${s.majorCode}`));
    pageNum++;
}

// Filter and count without creating a new array
const countCS = StudentDataset
    .fromGenerator(studentDatabase({ majorCode: 'CS' }))
    .count();

console.log(`\nNumber of CS students: `);

Example: Infinite Sequence with a Generator

// Fibonacci generator - infinite sequence
function* fibonacci() {
    let a = 0, b = 1;

    while (true) {
        yield a;
        [a, b] = [b, a + b];
    }
}

// Take the first 10 Fibonacci numbers
const fib = fibonacci();
const first10 = [];
for (let i = 0; i < 10; i++) {
    first10.push(fib.next().value);
}
console.log(first10); // [0, 1, 1, 2, 3, 5, 8, 13, 21, 34]

// Auto-increment ID generator
function* idGenerator(prefix = 'ID') {
    let id = 1;
    while (true) {
        yield `${prefix}-${String(id).padStart(5, '0')}`;
        id++;
    }
}

const ids = idGenerator('STU');
console.log(ids.next().value); // STU-00001
console.log(ids.next().value); // STU-00002

// Pagination generator for fetching data
async function* paginatedFetch(baseUrl, pageSize = 20) {
    let page = 1;
    let hasMore = true;

    while (hasMore) {
        const response = await fetch(`${baseUrl}?page=${page}&size=${pageSize}`);
        const data = await response.json();

        yield* data.items; // Delegate items

        hasMore = data.hasNextPage;
        page++;
    }
}

// Batch processor using a generator
function* batchProcessor(items, batchSize, processFn) {
    let batch = [];

    for (const item of items) {
        batch.push(item);

        if (batch.length === batchSize) {
            yield processFn(batch);
            batch = [];
        }
    }

    if (batch.length > 0) {
        yield processFn(batch);
    }
}

// Usage: batch insert into a database
const studentsToInsert = [
    { studentId: '001', name: 'Mike' },
    { studentId: '002', name: 'John' },
    { studentId: '003', name: 'Anna' },
    { studentId: '004', name: 'Diana' },
    { studentId: '005', name: 'Eric' }
];

for (const batch of batchProcessor(studentsToInsert, 2, (batch) => {
    console.log(`Inserting batch:`, batch.map(s => s.name));
    return batch;
})) {
    // Process the batch
}
// Output:
// Inserting batch: ['Mike', 'John']
// Inserting batch: ['Anna', 'Diana']
// Inserting batch: ['Eric']

Example: Async Data Pipeline

// Async generator for stream processing
async function* fetchStudentPages(baseUrl, pageSize = 10) {
    let page = 1;
    let hasMore = true;

    while (hasMore) {
        console.log(`Fetching page ${page}...`);

        const response = await fetch(`${baseUrl}?page=${page}&size=${pageSize}`);
        const data = await response.json();

        yield* data.students; // Yield each student

        hasMore = data.hasMore;
        page++;

        // Delay between pages to avoid overloading the server
        await new Promise(resolve => setTimeout(resolve, 100));
    }
}

// Async iterator with a filter
async function* filterAsync(asyncIter, predicate) {
    for await (const item of asyncIter) {
        if (predicate(item)) {
            yield item;
        }
    }
}

// Async iterator with a transform
async function* mapAsync(asyncIter, transform) {
    for await (const item of asyncIter) {
        yield transform(item);
    }
}

// Pipeline: fetch -> filter -> transform -> collect
async function getActiveCSStudents(baseUrl) {
    const results = [];

    const pipeline = mapAsync(
        filterAsync(
            fetchStudentPages(baseUrl),
            s => s.majorCode === 'CS' && s.status === 'active'
        ),
        s => ({
            name: s.name,
            studentId: s.studentId,
            average: s.grade.reduce((a, b) => a + b, 0) / s.grade.length
        })
    );

    for await (const student of pipeline) {
        results.push(student);
    }

    return results;
}

// Usage
const csStudents = await getActiveCSStudents('/api/students');
console.log(`Found ${csStudents.length} active CS students`);

Code Explanation

Iterator Protocol

• An object must have a next() method that returns { value, done }
• done: true signals that iteration is complete
• Can be iterated with for...of, spread [...], and destructuring

Symbol.iterator

• A special method that makes an object iterable
• Can be used with for...of and the spread operator
• Returns an iterator object with a next() method

Generator Functions

• Declared with function*
• yield to pause and return a value
• yield* to delegate to another iterable
• Returns an iterator that can be called with next()
• Can receive values through next(value)

Async Generators

• A combination of async function*
• yield works with Promises
• Can be iterated with for await...of
• Suited for data streaming or pagination

Conclusion

Iterators and generators are powerful features for lazy evaluation and sequence processing in JavaScript. With generators, you can process large datasets such as the data of an entire student cohort one item at a time without loading everything into memory. Async generators allow streaming data from a server efficiently. For a campus application, use generators for data pagination, batch processing, and lazy loading that keep performance optimal even with datasets of thousands of students.