Programming Challenge: Markov Chain Text Generator

Julia A bit basic but the results are already fun:

using StatsBase

tokenize(s) = lowercase.(split(replace(s,"."," .")," "))

function transistion_probablities(corpus)
    tokens = tokenize(corpus)

    d = Dict(zip(tokens,[String[] for i=1:length(tokens)]))
    for i=1:length(tokens)-1
        push!(d[tokens[i]], tokens[i+1])
    end

    Dict(zip(keys(d),[countmap(w) for w in values(d)])) #convert to counts
end

P(p::Dict{String,Int64}) = Weights(collect(values(p))) #convert to statistical weights

function generate(seed,α)
    state, out = seed, seed
    while state != "."
        state = StatsBase.sample(collect(keys(α[state])),P(α[state]))
        out = state == "." ? string(out, state) : string(out, " ", state)
    end
    out
end

corpus = read("epicurus.txt", String)
α = transistion_probablities(corpus)
generate("the",α)

Main> α = transistion_probablities(corpus)
Dict{String,Dict{String,Int64}} with 1867 entries:
"and,"       => Dict("if"=>1,"when"=>2,"since"=>1,"in"=>3)
"accord"     => Dict("with"=>1)
"action"     => Dict("of"=>2,"should"=>1)
"mutual"     => Dict("interconnection"=>1,"connection"=>1)
...

Main> generate("the",α)
"the objects from the atom which the properties which disappears had in order that is molded by sense."

Here's a quote image for this one, it's quite deep:

https://i.imgur.com/ibfS8mM.jpg

Python 3

Would be nice to handle punctuation better, but I'm too lazy.

#!/usr/bin/env python3

from random import choices, seed

from textblob.tokenizers import SentenceTokenizer, WordTokenizer

def ngrams(iterable, n=1):
    """Generate ngrams from an iterable

    l = range(5)
    list(l) -> [0, 1, 2, 3, 4, 5]
    list(ngrams(l, n=1)) -> [(0,), (1,), (2,), (3,), (4,)]
    list(ngrams(l, n=2)) -> [(0, 1), (1, 2), (2, 3), (3, 4)]
    list(ngrams(l, n=3)) -> [(0, 1, 2), (1, 2, 3), (2, 3, 4)]

    """
    return zip(*(iterable[i:] for i in range(n)))

def fdist(corpus, n=2):
    d = {}
    for sentence in SentenceTokenizer().tokenize(text=corpus):
        for ngram in ngrams(WordTokenizer().tokenize(text=sentence), n=n):
            first, *rest = ngram
            if first not in d:
                d[first] = {}
            for i, word in enumerate(rest):
                if i not in d[first]:
                    d[first][i] = {}
                if word not in d[first][i]:
                    d[first][i][word] = 0
                d[first][i][word] += 1
    return d

def generate_words(corpus, seedword, n=2):
    seed(seedword)
    fd = fdist(corpus, n=n)
    word = seedword
    yield word
    while word != '.' and word in fd:
        for i in fd[word]:
            population, frequencies = zip(*fd[word][i].items())
            weights = [f / len(frequencies) for f in frequencies]
            word, *_ = choices(population, weights)
            yield word

def generate_sentence(corpus, seedword, n=2):
    return ' '.join(generate_words(corpus, seedword, n=n))

if __name__ == '__main__':
    corpus = '''JABBERWOCKY
    By Lewis Carroll

    'Twas brillig, and the slithy toves
    Did gyre and gimble in the wabe;
    All mimsy were the borogoves,
    And the mome raths outgrabe.

    'Beware the Jabberwock, my son!
    The jaws that bite, the claws that catch!
    Beware the Jubjub bird, and shun
    The frumious Bandersnatch!'

    He took his vorpal sword in hand:
    Long time the manxome foe he sought--
    So rested he by the Tumtum tree,
    And stood awhile in thought.

    And as in uffish thought he stood,
    The Jabberwock, with eyes of flame,
    Came whiffling through the tulgey wood,
    And burbled as it came!

    One, two! One, two! And through and through
    The vorpal blade went snicker-snack!
    He left it dead, and with its head
    He went galumphing back.

    'And hast thou slain the Jabberwock?
    Come to my arms, my beamish boy!
    O frabjous day! Callooh! Callay!'
    He chortled in his joy.

    'Twas brillig, and the slithy toves
    Did gyre and gimble in the wabe;
    All mimsy were the borogoves,
    And the mome raths outgrabe.'''

    for seedword in ['He', 'Beware', 'One']:
        print(generate_sentence(corpus, seedword))

And the output:

./markov_chain.py 
He went snicker-snack ! '
Beware the wabe ; All mimsy were the slithy toves Did gyre and gimble in uffish thought he sought -- So rested he sought -- So rested he by the mome raths outgrabe .
One , and the claws that catch ! '